diff --git a/docs/features/search/collators.md b/docs/features/search/collators.md new file mode 100644 index 0000000000..4054ec1212 --- /dev/null +++ b/docs/features/search/collators.md @@ -0,0 +1,107 @@ +--- +id: collators +title: Collators +description: Indexing you Backstage content with Collators +--- + +Backstage includes 2 [collators](./concepts.md#collators) out of the box for the [Catalog](#catalog) and [TechDocs](#techdocs). There's also some from the [Backstage Community](#community-collators) too! + +## Catalog + +The Catalog collator will index all the Entities in your Catalog. It is installed by default but if you need to add it manually here's how. + +First we add the plugin into your backend app: + +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-search-backend-module-catalog +``` + +Then add the following line: + +```ts title="packages/backend/src/index.ts" +const backend = createBackend(); + +// Other plugins... + +// search plugin +backend.add(import('@backstage/plugin-search-backend/alpha')); + +/* highlight-add-start */ +backend.add(import('@backstage/plugin-search-backend-module-catalog/alpha')); +/* highlight-add-end */ + +backend.start(); +``` + +### Configuring the Catalog Collator + +The default schedule for the Catalog Collator is to run every 10 minutes, you can provide your own schedule by adding it to your config: + +```yaml title="app-config.yaml +search: + collators: + catalog: + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + initialDelay: { seconds: 90 } + # supports cron, ISO duration, "human duration" as used in code + frequency: { hours: 6 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +## TechDocs + +The TechDocs collator will index all the TechDocs in your Catalog. It is installed by default but if you need to add it manually here's how. + +First we add the plugin into your backend app: + +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-search-backend-module-techdocs +``` + +Then add the following line: + +```ts title="packages/backend/src/index.ts" +const backend = createBackend(); + +// Other plugins... + +// search plugin +backend.add(import('@backstage/plugin-search-backend/alpha')); + +/* highlight-add-start */ +backend.add(import('@backstage/plugin-search-backend-module-techdocs/alpha')); +/* highlight-add-end */ + +backend.start(); +``` + +### Configuring the TechDocs Collator + +The default schedule for the TechDocs Collator is to run every 10 minutes, you can provide your own schedule by adding it to your config: + +```yaml title="app-config.yaml +search: + collators: + techdocs: + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + initialDelay: { seconds: 90 } + # supports cron, ISO duration, "human duration" as used in code + frequency: { hours: 6 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +## Community Collators + +Here are some of the known Search Collators available in from the Backstage Community: + +- [`@backstage/plugin-search-backend-module-explore`](https://github.com/backstage/backstage/tree/master/plugins/search-backend-module-explore): will index content from the [Explore plugin](https://github.com/backstage/community-plugins/tree/main/workspaces/explore/plugins/explore). +- [`@backstage/plugin-search-backend-module-stack-overflow-collator`](https://github.com/backstage/backstage/tree/master/plugins/search-backend-module-stack-overflow-collator): will index content from Stack Overflow. +- [`@backstage-community/search-backend-module-adr`](https://github.com/backstage/community-plugins/tree/main/workspaces/adr/plugins/search-backend-module-adr): will index content from the [ADR plugin](https://github.com/backstage/community-plugins/tree/main/workspaces/adr/plugins/adr). + +## Custom Collators + +To create your own collators/decorators modules, please use the [searchModuleCatalogCollator](https://github.com/backstage/backstage/blob/d7f955f300893f50c4882ea8f5c09aa42dfaacfd/plugins/search-backend-module-catalog/src/alpha.ts#L49) as an example, we recommend that modules are separated by plugin packages (e.g. `search-backend-module-`). You can also find the available search engines and collator/decorator modules documentation in the Alpha API reports. diff --git a/docs/features/search/getting-started.md b/docs/features/search/getting-started.md index b2a074974f..c2ca2e1dce 100644 --- a/docs/features/search/getting-started.md +++ b/docs/features/search/getting-started.md @@ -133,79 +133,34 @@ For more information about using `Root.tsx`, please see Add the following plugins into your backend app: ```bash title="From your Backstage root directory" -yarn --cwd packages/backend add @backstage/plugin-search-backend @backstage/plugin-search-backend-node +yarn --cwd packages/backend add @backstage/plugin-search-backend @backstage/plugin-search-backend-module-pg @backstage/plugin-search-backend-module-catalog @backstage/plugin-search-backend-module-techdocs ``` -Create a `packages/backend/src/plugins/search.ts` file containing the following -code: +Then add the following lines: -```typescript -import { useHotCleanup } from '@backstage/backend-common'; -import { createRouter } from '@backstage/plugin-search-backend'; -import { - IndexBuilder, - LunrSearchEngine, -} from '@backstage/plugin-search-backend-node'; -import { PluginEnvironment } from '../types'; -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; -import { Router } from 'express'; +```ts title="packages/backend/src/index.ts" +const backend = createBackend(); -export default async function createPlugin( - env: PluginEnvironment, -): Promise { - const searchEngine = new LunrSearchEngine({ - logger: env.logger, - }); - const indexBuilder = new IndexBuilder({ - logger: env.logger, - searchEngine, - }); +// Other plugins... - const every10MinutesSchedule = env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 10 }, - timeout: { minutes: 15 }, - initialDelay: { seconds: 3 }, - }); +/* highlight-add-start */ +// search plugin +backend.add(import('@backstage/plugin-search-backend/alpha')); - indexBuilder.addCollator({ - schedule: every10MinutesSchedule, - factory: DefaultCatalogCollatorFactory.fromConfig(env.config, { - discovery: env.discovery, - tokenManager: env.tokenManager, - }), - }); +// search engines +backend.add(import('@backstage/plugin-search-backend-module-pg/alpha')); - const { scheduler } = await indexBuilder.build(); +// search collators +backend.add(import('@backstage/plugin-search-backend-module-catalog/alpha')); +backend.add(import('@backstage/plugin-search-backend-module-techdocs/alpha')); +/* highlight-add-end */ - scheduler.start(); - useHotCleanup(module, () => scheduler.stop()); - - return await createRouter({ - engine: indexBuilder.getSearchEngine(), - logger: env.logger, - }); -} +backend.start(); ``` -Make the following modifications to your `packages/backend/src/index.ts` file: +With the above setup Search will use the [Lunr](https://github.com/olivernn/lunr.js) in-memory Search Engine but if your have Postgres setup as your database then it will use Postgres as your Search Engine. Learn more in the [Search Engines](./search-engines.md) documentation. -Import the `plugins/search` file you created above: - -```typescript -import search from './plugins/search'; -``` - -Set up an environment for search: - -```typescript -const searchEnv = useHotMemoize(module, () => createEnv('search')); -``` - -Register the search service with the router: - -```typescript -apiRouter.use('/search', await search(searchEnv)); -``` +The above also sets up two Collators for you - Catalog and TechDocs - which will index content from these two locations so that you can easily search them. Learn more in the [Collators documentation](./collators.md). ## Customizing Search diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index 9ce30da556..b0df9440f3 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -45,75 +45,6 @@ to do that in two steps. }); ``` -## How to index TechDocs documents - -The TechDocs plugin has supported integrations to Search, meaning that it -provides a default collator factory ready to be used. - -The purpose of this guide is to walk you through how to register the -[DefaultTechDocsCollatorFactory](https://github.com/backstage/backstage/blob/1adc2c7/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts) -in your App, so that you can get TechDocs documents indexed. - -If you have been through the -[Getting Started with Search guide](https://backstage.io/docs/features/search/getting-started), -you should have the `packages/backend/src/plugins/search.ts` file available. If -so, you can go ahead and follow this guide - if not, start by going through the -getting started guide. - -1. Import the `DefaultTechDocsCollatorFactory` from - `@backstage/plugin-search-backend-module-techdocs`. - - ```typescript - import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-search-backend-module-techdocs'; - ``` - -2. If there isn't an existing schedule you'd like to run the collator on, be - sure to create it first. Something like... - - ```typescript - import { Duration } from 'luxon'; - - const every10MinutesSchedule = env.scheduler.createScheduledTaskRunner({ - frequency: Duration.fromObject({ seconds: 600 }), - timeout: Duration.fromObject({ seconds: 900 }), - initialDelay: Duration.fromObject({ seconds: 3 }), - }); - ``` - -3. Register the `DefaultTechDocsCollatorFactory` with the IndexBuilder. - - ```typescript - indexBuilder.addCollator({ - schedule: every10MinutesSchedule, - factory: DefaultTechDocsCollatorFactory.fromConfig(env.config, { - discovery: env.discovery, - logger: env.logger, - tokenManager: env.tokenManager, - }), - }); - ``` - -You should now have your TechDocs documents indexed to your search engine of -choice! - -If you want your users to be able to filter down to the techdocs type when -searching, you can update your `SearchPage.tsx` file in -`packages/app/src/components/search` by adding `techdocs` to the list of values -of the `SearchType` component. - -```tsx title="packages/app/src/components/search/SearchPage.tsx" - - - {/* ... */} - -``` - -> Check out the documentation around [integrating search into plugins](../../plugins/integrating-search-into-plugins.md#create-a-collator) for how to create your own collator. - ## How to customize fields in the Software Catalog or TechDocs index Sometimes, you might want to have the ability to control which data passes into the search index @@ -388,38 +319,3 @@ export const SearchModal = ({ toggleModal }: { toggleModal: () => void }) => ( ``` There are other more specific search results layout components that also accept result item extensions, check their documentation: [SearchResultList](https://backstage.io/storybook/?path=/story/plugins-search-searchresultlist--with-result-item-extensions) and [SearchResultGroup](https://backstage.io/storybook/?path=/story/plugins-search-searchresultgroup--with-result-item-extensions). - -## How to migrate your backend installation to use Search together with the new backend system - -Recently, the Backstage maintainers [announced the new Backend System](https://backstage.io/blog/2023/02/15/backend-system-alpha). The search plugins are now migrated to support the new backend system. In this guide you will learn how to update your backend set up. - -In "packages/backend/index.ts", install the search plugin [1], the search engine [2], and the search collators/decorators modules [3]: - -```ts -const backend = createBackend(); -// [1] adding the search plugin to the backend -backend.add(import('@backstage/plugin-search-backend/alpha')); -// [2] (optional) the default search engine is Lunr, if you want to extend the search backend with another search engine. -backend.add( - import('@backstage/plugin-search-backend-module-elasticsearch/alpha'), -); -// [3] extending search with collator modules to start index documents, take in optional schedule parameters. -backend.add(import('@backstage/plugin-search-backend-module-catalog/alpha')); -backend.add(import('@backstage/plugin-search-backend-module-techdocs/alpha')); -backend.add(import('@backstage/plugin-search-backend-module-explore/alpha')); - -backend.start(); -``` - -To create your own collators/decorators modules, please use the [searchModuleCatalogCollator](https://github.com/backstage/backstage/blob/d7f955f300893f50c4882ea8f5c09aa42dfaacfd/plugins/search-backend-module-catalog/src/alpha.ts#L49) as an example, we recommend that modules are separated by plugin packages (e.g. `search-backend-module-`). You can also find the available search engines and collator/decorator modules documentation in the Alpha API reports: - -**Search engine modules** - -- Postgres [module](https://github.com/backstage/backstage/blob/d7f955f300893f50c4882ea8f5c09aa42dfaacfd/plugins/search-backend-module-pg/alpha-api-report.md); -- Elasticsearch [module](https://github.com/backstage/backstage/blob/d7f955f300893f50c4882ea8f5c09aa42dfaacfd/plugins/search-backend-module-elasticsearch/alpha-api-report.md). - -**Search collator/decorator modules** - -- Catalog [module](https://github.com/backstage/backstage/blob/d7f955f300893f50c4882ea8f5c09aa42dfaacfd/plugins/search-backend-module-catalog/alpha-api-report.md); -- Explore [module](https://github.com/backstage/backstage/blob/d7f955f300893f50c4882ea8f5c09aa42dfaacfd/plugins/search-backend-module-explore/alpha-api-report.md); -- TechDocs [module](https://github.com/backstage/backstage/blob/d7f955f300893f50c4882ea8f5c09aa42dfaacfd/plugins/search-backend-module-techdocs/alpha-api-report.md). diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index 86b6244fb6..f768a9bde8 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -4,34 +4,31 @@ title: Search Engines description: Choosing and configuring your search engine for Backstage --- -Backstage supports 3 search engines by default, an in-memory engine called Lunr, -Elasticsearch and Postgres. You can configure your own search engines by -implementing the provided interface as mentioned in the -[search backend documentation.](./getting-started.md#Backend) - -Provided search engine implementations have their own way of constructing -queries, which may be something you want to modify. Alterations to the querying -logic of a search engine can be made by providing your own implementation of a -QueryTranslator interface. This modification can be done without touching -provided search engines by using the exposed setter to set the modified query -translator into the instance. - -```typescript -const searchEngine = new LunrSearchEngine({ logger: env.logger }); -searchEngine.setTranslator(new MyNewAndBetterQueryTranslator()); -``` +Backstage supports 3 search engines by default, an in-memory engine called [Lunr](#lunr), [Postgres](#postgres) +and [Elasticsearch](#elasticsearch). ## Lunr -Lunr search engine is enabled by default for your backstage instance if you have -not done additional changes to the scaffolded app. +Lunr search engine is enabled by default for your Backstage instance if you have not done additional changes to the scaffolded app. -Lunr can be instantiated like this: +As Lunr is built into the Search backend plugin it can be added like this: -```typescript -// app/backend/src/plugins/search.ts -const searchEngine = new LunrSearchEngine({ logger: env.logger }); -const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine }); +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-search-backend +``` + +Then add the following line: + +```ts title="packages/backend/src/index.ts" +const backend = createBackend(); + +// Other plugins... + +/* highlight-add-start */ +backend.add(import('@backstage/plugin-search-backend/alpha')); +/* highlight-add-end */ + +backend.start(); ``` :::note Note @@ -45,34 +42,39 @@ other search engines instead. ## Postgres -The Postgres based search engine only requires that postgres being configured as +The Postgres based search engine only requires that Postgres being configured as the database engine for Backstage. Therefore it targets setups that want to avoid maintaining another external service like Elasticsearch. The search provides decent results and performs well with ten thousands of indexed -documents. The connection to postgres is established via the database manager +documents. The connection to Postgres is established via the database manager also used by other plugins. > **Important**: The search plugin requires at least Postgres 12! -To use the `PgSearchEngine`, make sure that you have a Postgres database -configured and make the following changes to your backend: +First we need to add the plugin: -1. Add a dependency on `@backstage/plugin-search-backend-module-pg` to your - backend's `package.json`. -2. Initialize the search engine. It is recommended to initialize it with a - fallback to the lunr search engine if you are running Backstage for - development locally with SQLite: - -```typescript -// In packages/backend/src/plugins/search.ts - -// Initialize a connection to a search engine. -const searchEngine = (await PgSearchEngine.supported(env.database)) - ? await PgSearchEngine.fromConfig(env.config, { database: env.database }) - : new LunrSearchEngine({ logger: env.logger }); +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-search-backend-module-pg ``` -## Optional Configuration +Then add the following line: + +```ts title="packages/backend/src/index.ts" +const backend = createBackend(); + +// Other plugins... + +// search plugin +backend.add(import('@backstage/plugin-search-backend/alpha')); + +/* highlight-add-start */ +backend.add(import('@backstage/plugin-search-backend-module-pg/alpha')); +/* highlight-add-end */ + +backend.start(); +``` + +### Optional Configuration The following is an example of the optional configuration that can be applied when using Postgres as the search backend. Currently this is mostly for just the highlight feature: @@ -106,19 +108,32 @@ Backstage supports Elasticsearch (and OpenSearch) search engine connections, indexing and querying out of the box. Available configuration options enable usage of either AWS or Elastic.co hosted solutions, or a custom self-hosted solution. -Similarly to Lunr above, Elasticsearch can be set up like this: +Similarly to Postgres above, Elasticsearch can be set up as follows. -```typescript -// app/backend/src/plugins/search.ts -const searchEngine = await ElasticSearchSearchEngine.fromConfig({ - logger: env.logger, - config: env.config, -}); -const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine }); +First we need to add the plugin: + +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-search-backend-module-elasticsearch ``` -For the engine to be available, your backend package needs a dependency on -package `@backstage/plugin-search-backend-module-elasticsearch`. +Then add the following line: + +```ts title="packages/backend/src/index.ts" +const backend = createBackend(); + +// Other plugins... + +// search plugin +backend.add(import('@backstage/plugin-search-backend/alpha')); + +/* highlight-add-start */ +backend.add( + import('@backstage/plugin-search-backend-module-elasticsearch/alpha'), +); +/* highlight-add-end */ + +backend.start(); +``` Elasticsearch needs some additional configuration before it is ready to use within your instance. The configuration options are documented in the @@ -126,67 +141,12 @@ within your instance. The configuration options are documented in the The underlying functionality uses either the official Elasticsearch client version 7.x (meaning that Elasticsearch version 7 is the only one confirmed to -be supported), or the OpenSearch client, when the `aws` or `opensearch `provider +be supported), or the OpenSearch client, when the `aws` or `opensearch` provider is configured. -Should you need to create your own bespoke search experiences that require more -than just a query translator (such as faceted search or Relay pagination), you -can access the configuration of the search engine in order to create new -Elasticsearch clients. The version of the client need not be the same as one -used internally by the Elasticsearch engine plugin. For example: +### Example configurations -```typescript -import { isOpenSearchCompatible } from '@backstage/plugin-search-backend-module-elasticsearch'; -import { Client as ElasticClient } from '@elastic/elasticsearch'; -import { Client as OpenSearchClient } from '@opensearch-project/opensearch'; - -// Return an Elasticsearch client -const esClient = searchEngine.newClient(options => { - if (!isOpenSearchCompatible(options)) { - return new ElasticClient(options); - } - - throw new Error('Incompatible options'); -}); - -// Return an OpenSearch client -const osClient = searchEngine.newClient(options => { - if (isOpenSearchCompatible(options)) { - return new OpenSearchClient(options); - } - - throw new Error('Incompatible options'); -}); -``` - -#### Set custom index template - -The Elasticsearch engine gives you the ability to set a custom index template if needed. - -> Index templates define settings, mappings, and aliases that can be applied automatically to new indices. - -```typescript -// app/backend/src/plugins/search.ts -const searchEngine = await ElasticSearchSearchEngine.initialize({ - logger: env.logger, - config: env.config, -}); - -searchEngine.setIndexTemplate({ - name: '', - body: { - index_patterns: [''], - template: { - mappings: {}, - settings: {}, - }, - }, -}); -``` - -## Example configurations - -### AWS +#### AWS Using AWS hosted Elasticsearch the only configuration option needed is the URL to the Elasticsearch service. The implementation assumes that environment @@ -201,7 +161,7 @@ search: node: https://my-backstage-search-asdfqwerty.eu-west-1.es.amazonaws.com ``` -### Elastic.co +#### Elastic.co Elastic Cloud hosted Elasticsearch uses a Cloud ID to determine the instance of hosted Elasticsearch to connect to. Additionally, username and password needs to @@ -218,7 +178,7 @@ search: password: changeme ``` -### OpenSearch +#### OpenSearch OpenSearch can be self hosted for example with the [official docker image](https://hub.docker.com/r/opensearchproject/opensearch). The configuration requires only the node and authentication. @@ -232,7 +192,7 @@ search: password: changeme ``` -### Others +#### Others Other Elasticsearch instances can be connected to by using standard Elasticsearch authentication methods and exposed URL, provided that the cluster @@ -242,8 +202,6 @@ username/password or an API key. For more information how to create an API key, see [Elastic documentation on API keys](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html). -#### Configuration examples - ##### With username and password ```yaml @@ -273,8 +231,6 @@ you may get an error caused by limited `thread_pool` configuration. ( `429 Too M In this case you need to decrease the batch size to index the resources to prevent this kind of error. You can easily decrease or increase the batch size in your `app-config.yaml` using the `batchSize` option provided for Elasticsearch configuration. -#### Configuration example - **Set batch size to 100** ```yaml diff --git a/microsite/sidebars.json b/microsite/sidebars.json index ddccb14c91..ac735880bf 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -134,6 +134,7 @@ "features/search/concepts", "features/search/architecture", "features/search/search-engines", + "features/search/collators", "features/search/how-to-guides" ] }, diff --git a/mkdocs.yml b/mkdocs.yml index 3b0c7d9caf..33c9af3286 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -71,6 +71,7 @@ nav: - Concepts: 'features/search/concepts.md' - Search Architecture: 'features/search/architecture.md' - Search Engines: 'features/search/search-engines.md' + - Collators: 'features/search/collators.md' - HOW TO guides: 'features/search/how-to-guides.md' - TechDocs: - Overview: 'features/techdocs/README.md'