From 92a37a8fc71edbe8fc3685c5aee1a16468278683 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 5 Jan 2023 08:28:48 +0100 Subject: [PATCH 1/6] chore: bump minimum required version of better-sqlite Signed-off-by: blam --- .../templates/default-app/packages/backend/package.json.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-app/templates/default-app/packages/backend/package.json.hbs b/packages/create-app/templates/default-app/packages/backend/package.json.hbs index ca644265b8..cb358db911 100644 --- a/packages/create-app/templates/default-app/packages/backend/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/backend/package.json.hbs @@ -34,7 +34,7 @@ "@backstage/plugin-search-backend-node": "^{{version '@backstage/plugin-search-backend-node'}}", "@backstage/plugin-techdocs-backend": "^{{version '@backstage/plugin-techdocs-backend'}}", "app": "link:../app", - "better-sqlite3": "^7.5.0", + "better-sqlite3": "^8.0.0", "dockerode": "^3.3.1", "express": "^4.17.1", "express-promise-router": "^4.1.0", From 016a52c28f5b1e75ebde6635777a5b7d1173dd29 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 5 Jan 2023 08:31:02 +0100 Subject: [PATCH 2/6] chre: changeset Signed-off-by: blam --- .changeset/wicked-socks-build.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/wicked-socks-build.md diff --git a/.changeset/wicked-socks-build.md b/.changeset/wicked-socks-build.md new file mode 100644 index 0000000000..96572185b0 --- /dev/null +++ b/.changeset/wicked-socks-build.md @@ -0,0 +1,10 @@ +--- +'@backstage/create-app': patch +--- + +Bumped the minimum version of `better-sqlite3`. You can make the following change to your `packages/backend/package.json` to keep your instance of Backstage updated. + +```diff +- "better-sqlite3": "^7.5.0", ++ "better-sqlite3": "^8.0.0", +``` From 5e8cd3c72cd5ce54bd0f83f68c3bc8b45c50ca6a Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 5 Jan 2023 14:30:54 +0100 Subject: [PATCH 3/6] Clean up search docs homepage Signed-off-by: Eric Peterson --- docs/features/search/README.md | 82 +++------------------------------- 1 file changed, 7 insertions(+), 75 deletions(-) diff --git a/docs/features/search/README.md b/docs/features/search/README.md index 21deffb3a0..97d3d7efcb 100644 --- a/docs/features/search/README.md +++ b/docs/features/search/README.md @@ -23,7 +23,8 @@ See the more detailed [architecture](./architecture.md) and [tech stack](./archi ## Project roadmap -No current plans. +No current plans. Check [Backstage issues labeled `search`](https://github.com/backstage/backstage/issues?q=is%3Aopen+is%3Aissue+label%3Asearch) +for community-led ideas and initiatives. ## Supported @@ -42,11 +43,11 @@ The following sections show the plugins and search engines currently supported b See [Backstage Search Architecture](architecture.md) to get an overview of how the search engines are used. -| Search Engines | Support Status | -| -------------------------------------------------- | -------------- | -| [ElasticSearch](./search-engines.md#elasticsearch) | ✅ | -| [Lunr](./search-engines.md#lunr) | ✅ | -| [Postgres](./search-engines.md#postgres) | Community ✅ | +| Search Engines | Support Status | +| ------------------------------------------------------------- | -------------- | +| [Elasticsearch/OpenSearch](./search-engines.md#elasticsearch) | ✅ | +| [Lunr](./search-engines.md#lunr) | ✅ | +| [Postgres](./search-engines.md#postgres) | Community ✅ | [Reach out to us](#get-involved) if you want to chat about support for more plugin integrations and search engines. @@ -56,72 +57,3 @@ search engines. For any questions, feedback, or to help move search forward, reach out to us in the **#search** channel of our [Discord chatroom](https://github.com/backstage/backstage#community). - -## Use Cases supported - -#### **Backstage Search Pre-Alpha** - -Search Frontend letting you search through the entities of the software catalog. - -The pre-alpha is intended to solve for the following user stories, but will get -there by means of a front-end only, non-extensible MVP. - -- As a software engineer I should be able to navigate to a search page and - search for entities registered in the Software Catalog. -- As a software engineer I should be able to use the search input field in the - sidebar to search for entities registered in the Software Catalog. -- As a software engineer I should be able to see the number of results my search - returned. -- As a software engineer I should be able to filter on metadata (kind, - lifecycle) when I’ve performed a search. -- As a software engineer I should be able to hide the filters if I don’t need to - use them. - -#### **Backstage Search Alpha** - -Basic “out-of-the-box” in-memory indexing process of entities, and their metadata, registered to the Software Catalog. - -We will consider Backstage Search to be in alpha when the above use-cases are -met, but built on top of a flexible, extensible platform. - -- As an integrator, I should be able to provide all of the pre-alpha experiences - to my users if I choose, but also be able to customize the experience using a - composable set of components. -- As a plugin developer, I should have a standard way to expose my plugin's data - to Backstage Search. -- As an integrator, I should still be able to expose everything in the Software - Catalog in search, but it should be possible to customize what is searchable. -- As an integrator, although I should be able to customize all of the above, it - should be possible to have the pre-alpha user experiences covered without - having to set up and configure a search engine. - -#### **Backstage Search Beta** - -At least one production-ready search engine that supports the same use-cases as in the alpha. - -We will consider Backstage Search to be in a beta phase when the above use-cases -are met, and can be deployed using a production-ready search engine. - -- As an integrator, I should be able to power my Backstage Search experience - (including querying and indexing) using a production-ready search engine like - ElasticSearch. -- As an integrator, I should be able to configure the connection to my search - engine in **app_config.yaml**. -- As an integrator, I should be able to tune the queries sent to my chosen - search engine according to my organization's needs, but a sensible default - query should be in place so that I am not required to do so. - -#### **Backstage Search 1.0** - -A stable Search API for plugin developers to add search to their plugins, and app integrators to expose that to their users. - -We will consider Backstage Search to be 1.0 when the above -use-cases are met, and an ecosystem of search-enabled plugins are available and -stable. - -- As a plugin developer, there should be at least one example of a Backstage - plugin that integrates with search that I can use as inspiration for my own - plugin's search capabilities (for example, the TechDocs plugin). -- As an app integrator, there should be plenty of examples and documentation on - how to customize and extend search in my Backstage instance to meet my - organization's needs. From c2f1041c152d589556a06d3fa719c251f3af8f9b Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 5 Jan 2023 14:32:26 +0100 Subject: [PATCH 4/6] Note limitation about Lunr in production Signed-off-by: Eric Peterson --- docs/features/search/search-engines.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index ccd8268dbe..68df3f0944 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -34,6 +34,11 @@ const searchEngine = new LunrSearchEngine({ logger: env.logger }); const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine }); ``` +> Note: Lunr is appropriate as a zero-config search engine when developing +> other parts of Backstage locally, however its use is highly discouraged when +> running Backstage in production. When deploying Backstage, use one of the +> other search engines instead. + ## Postgres The Postgres based search engine only requires that postgres being configured as From 503359280e50861f19eb7e7f423c4e54fdc1b234 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 5 Jan 2023 14:33:15 +0100 Subject: [PATCH 5/6] Use correct case when referring to Elasticsearch. Signed-off-by: Eric Peterson --- docs/features/search/concepts.md | 2 +- docs/features/search/getting-started.md | 2 +- docs/features/search/search-engines.md | 46 ++++++++++++------------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/features/search/concepts.md b/docs/features/search/concepts.md index 57ab061974..69a7223085 100644 --- a/docs/features/search/concepts.md +++ b/docs/features/search/concepts.md @@ -23,7 +23,7 @@ To get started, you should get familiar with these core concepts: Backstage Search isn't a search engine itself, rather, it provides an interface between your Backstage instance and a Search Engine of your choice. More concretely, a `SearchEngine` is an interface whose concrete implementations -facilitate communication with different search engines (like ElasticSearch, +facilitate communication with different search engines (like Elasticsearch, Lunr, Solr, etc). This abstraction exists in order to support your organization's needs. diff --git a/docs/features/search/getting-started.md b/docs/features/search/getting-started.md index e13401a8e3..1787189e91 100644 --- a/docs/features/search/getting-started.md +++ b/docs/features/search/getting-started.md @@ -286,7 +286,7 @@ an example: Backstage Search isn't a search engine itself, rather, it provides an interface between your Backstage instance and a [Search Engine](./concepts.md#search-engines) of your choice. Currently, we only -support two engines, an in-memory search Engine called Lunr and ElasticSearch. +support two engines, an in-memory search Engine called Lunr and Elasticsearch. See [Search Engines](./search-engines.md) documentation for more information how to configure these in your Backstage instance. diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index 68df3f0944..2e7341a0d7 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -5,7 +5,7 @@ description: Choosing and configuring your search engine for Backstage --- Backstage supports 3 search engines by default, an in-memory engine called Lunr, -ElasticSearch and Postgres. You can configure your own search engines by +Elasticsearch and Postgres. You can configure your own search engines by implementing the provided interface as mentioned in the [search backend documentation.](./getting-started.md#Backend) @@ -43,7 +43,7 @@ const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine }); The Postgres based search engine only requires that postgres being configured as the database engine for Backstage. Therefore it targets setups that want to -avoid maintaining another external service like elastic search. The search +avoid maintaining another external service like Elasticsearch. The search provides decent results and performs well with ten thousands of indexed documents. The connection to postgres is established via the database manager also used by other plugins. @@ -96,13 +96,13 @@ search: The Postgres documentation on [Highlighting Results](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE) has more details. -## ElasticSearch +## Elasticsearch -Backstage supports ElasticSearch search engine connections, indexing and -querying out of the box. Available configuration options enable usage of either -AWS or Elastic.co hosted solutions, or a custom self-hosted solution. +Backstage supports Elasticsearch (and OpenSearch) search engine connections, +indexing and querying out of the box. Available configuration options enable +usage of either AWS or Elastic.co hosted solutions, or a custom self-hosted solution. -Similarly to Lunr above, ElasticSearch can be set up like this: +Similarly to Lunr above, Elasticsearch can be set up like this: ```typescript // app/backend/src/plugins/search.ts @@ -116,20 +116,20 @@ const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine }); For the engine to be available, your backend package needs a dependency on package `@backstage/plugin-search-backend-module-elasticsearch`. -ElasticSearch needs some additional configuration before it is ready to use +Elasticsearch needs some additional configuration before it is ready to use within your instance. The configuration options are documented in the [configuration schema definition file.](https://github.com/backstage/backstage/blob/master/plugins/search-backend-module-elasticsearch/config.d.ts) -The underlying functionality uses either the official ElasticSearch client -version 7.x (meaning that ElasticSearch version 7 is the only one confirmed to +The underlying functionality uses either the official Elasticsearch client +version 7.x (meaning that Elasticsearch version 7 is the only one confirmed to be supported), or the OpenSearch client, when the `aws` or `opensearch `provider is configured. Should you need to create your own bespoke search experiences that require more than just a query translator (such as faceted search or Relay pagination), you -can access the configuration of the search engine in order to create new elastic -search clients. The version of the client need not be the same as one used -internally by the elastic search engine plugin. For example: +can access the configuration of the search engine in order to create new +Elasticsearch clients. The version of the client need not be the same as one +used internally by the Elasticsearch engine plugin. For example: ```typescript import { isOpenSearchCompatible } from '@backstage/plugin-search-backend-module-elasticsearch'; @@ -149,7 +149,7 @@ const client = searchEngine.newClient(options => { #### Set custom index template -The elasticsearch engine gives you the ability to set a custom index template if needed. +The Elasticsearch engine gives you the ability to set a custom index template if needed. > Index templates define settings, mappings, and aliases that can be applied automatically to new indices. @@ -176,8 +176,8 @@ searchEngine.setIndexTemplate({ ### AWS -Using AWS hosted ElasticSearch the only configuration option needed is the URL -to the ElasticSearch service. The implementation assumes that environment +Using AWS hosted Elasticsearch the only configuration option needed is the URL +to the Elasticsearch service. The implementation assumes that environment variables for AWS access key id and secret access key are defined in accordance to the [default AWS credential chain.](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html). @@ -191,8 +191,8 @@ search: ### Elastic.co -Elastic Cloud hosted ElasticSearch uses a Cloud ID to determine the instance of -hosted ElasticSearch to connect to. Additionally, username and password needs to +Elastic Cloud hosted Elasticsearch uses a Cloud ID to determine the instance of +hosted Elasticsearch to connect to. Additionally, username and password needs to be provided either directly or using environment variables like defined in [Backstage documentation.](https://backstage.io/docs/conf/writing#includes-and-dynamic-data) @@ -222,8 +222,8 @@ search: ### Others -Other ElasticSearch instances can be connected to by using standard -ElasticSearch authentication methods and exposed URL, provided that the cluster +Other Elasticsearch instances can be connected to by using standard +Elasticsearch authentication methods and exposed URL, provided that the cluster supports that. The configuration options needed are the URL to the node and authentication information. Authentication can be handled by either providing username/password or an API key. For more information how to create an API key, @@ -253,13 +253,13 @@ search: apiKey: base64EncodedKey ``` -### Elastic search batch size +### Elasticsearch batch size -Default batch size of the elastic search engine is set to 1000. If you are using a lower spec computing resources (like AWS small instance), +Default batch size of the Elasticsearch engine is set to 1000. If you are using a lower spec computing resources (like AWS small instance), you may get an error caused by limited `thread_pool` configuration. ( `429 Too Many Requests /_bulk` ) In this case you need to decrease the batch size to index the resources to prevent this kind of error. You can easily decrease -or increase the batch size in your `app-config.yaml` using the `batchSize` option provided for elasticsearch configuration. +or increase the batch size in your `app-config.yaml` using the `batchSize` option provided for Elasticsearch configuration. #### Configuration example From 6387ec870657a700026db749db46d6d6fa6162d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 5 Jan 2023 17:27:54 +0100 Subject: [PATCH 6/6] Revert "chore(deps): update dependency puppeteer to v19" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- packages/e2e-test/package.json | 2 +- packages/e2e-test/src/lib/helpers.ts | 2 +- yarn.lock | 63 ++++++++++++---------------- 3 files changed, 29 insertions(+), 38 deletions(-) diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json index 6bef34d633..374eaa6d18 100644 --- a/packages/e2e-test/package.json +++ b/packages/e2e-test/package.json @@ -40,7 +40,7 @@ "handlebars": "^4.7.3", "nodemon": "^2.0.2", "pgtools": "^0.3.0", - "puppeteer": "^19.0.0", + "puppeteer": "^17.0.0", "tree-kill": "^1.2.2", "ts-node": "^10.0.0" }, diff --git a/packages/e2e-test/src/lib/helpers.ts b/packages/e2e-test/src/lib/helpers.ts index 2e9b355f4c..b3fd665fd2 100644 --- a/packages/e2e-test/src/lib/helpers.ts +++ b/packages/e2e-test/src/lib/helpers.ts @@ -22,7 +22,7 @@ import { ChildProcess, } from 'child_process'; import { promisify } from 'util'; -import * as puppeteer from 'puppeteer'; +import puppeteer from 'puppeteer'; const execFile = promisify(execFileCb); diff --git a/yarn.lock b/yarn.lock index a47f5bad21..eed39c16fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19495,18 +19495,6 @@ __metadata: languageName: node linkType: hard -"cosmiconfig@npm:8.0.0": - version: 8.0.0 - resolution: "cosmiconfig@npm:8.0.0" - dependencies: - import-fresh: ^3.2.1 - js-yaml: ^4.1.0 - parse-json: ^5.0.0 - path-type: ^4.0.0 - checksum: ff4cdf89ac1ae52e7520816622c21a9e04380d04b82d653f5139ec581aa4f7f29e096d46770bc76c4a63c225367e88a1dfa233ea791669a35101f5f9b972c7d1 - languageName: node - linkType: hard - "cosmiconfig@npm:^6.0.0": version: 6.0.0 resolution: "cosmiconfig@npm:6.0.0" @@ -20689,10 +20677,10 @@ __metadata: languageName: node linkType: hard -"devtools-protocol@npm:0.0.1068969": - version: 0.0.1068969 - resolution: "devtools-protocol@npm:0.0.1068969" - checksum: 53b9c8d661e4148eaf8e990f03902fb3a2cceb06044f661013b6c92dd48ece397ef49fd18401775823c9a33069b4b535502f2559d4f99c74a6bdcb71582b6c8a +"devtools-protocol@npm:0.0.1036444": + version: 0.0.1036444 + resolution: "devtools-protocol@npm:0.0.1036444" + checksum: 6975c8def95a5e1a4207d6deb05322e335d6a37bdaa3e589cbd5bde40fbbe3ab0df2cfedb1b3ad2785401f208c150fd489a6a065a4624b56e4c0c4c1bfd89172 languageName: node linkType: hard @@ -21072,7 +21060,7 @@ __metadata: handlebars: ^4.7.3 nodemon: ^2.0.2 pgtools: ^0.3.0 - puppeteer: ^19.0.0 + puppeteer: ^17.0.0 tree-kill: ^1.2.2 ts-node: ^10.0.0 bin: @@ -32804,34 +32792,22 @@ __metadata: languageName: node linkType: hard -"puppeteer-core@npm:19.4.1": - version: 19.4.1 - resolution: "puppeteer-core@npm:19.4.1" +"puppeteer@npm:^17.0.0": + version: 17.1.3 + resolution: "puppeteer@npm:17.1.3" dependencies: cross-fetch: 3.1.5 debug: 4.3.4 - devtools-protocol: 0.0.1068969 + devtools-protocol: 0.0.1036444 extract-zip: 2.0.1 https-proxy-agent: 5.0.1 + progress: 2.0.3 proxy-from-env: 1.1.0 rimraf: 3.0.2 tar-fs: 2.1.1 unbzip2-stream: 1.4.3 - ws: 8.11.0 - checksum: f4db1aa1d1e642356744bed2dc49ff7960a8b530d147ace1fe067937433ae40234ea973260b9b264d53a2f2297f53df796d008d4756f735b3f0a64a69724eaba - languageName: node - linkType: hard - -"puppeteer@npm:^19.0.0": - version: 19.4.1 - resolution: "puppeteer@npm:19.4.1" - dependencies: - cosmiconfig: 8.0.0 - https-proxy-agent: 5.0.1 - progress: 2.0.3 - proxy-from-env: 1.1.0 - puppeteer-core: 19.4.1 - checksum: 14b6bd8d5f73f389e5bf424df6a89c346b7a8a8d0326b6aa3670e447a98b638b9091f46b6980515d56152b916a4e7be49c21f25fe7bb5c66828e02466a891f57 + ws: 8.8.1 + checksum: b4518956c661df4c37690d46b9c6744d42d59d01fe3764938b4b3af8de4c94ef11a9dfa6bc261798f5e5490c6dce4d4f555f05d42c735249683da3017faf4585 languageName: node linkType: hard @@ -38816,6 +38792,21 @@ __metadata: languageName: node linkType: hard +"ws@npm:8.8.1": + version: 8.8.1 + resolution: "ws@npm:8.8.1" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: 2152cf862cae0693f3775bc688a6afb2e989d19d626d215e70f5fcd8eb55b1c3b0d3a6a4052905ec320e2d7734e20aeedbf9744496d62f15a26ad79cf4cf7dae + languageName: node + linkType: hard + "ws@npm:^5.2.0 || ^6.0.0 || ^7.0.0, ws@npm:^7.3.1, ws@npm:^7.4.6": version: 7.5.9 resolution: "ws@npm:7.5.9"