feat(search): handle search indexing coordination among nodes

Signed-off-by: Phil Kuang <pkuang@factset.com>

docs/features/search/concepts.md

@@ -84,7 +84,9 @@ index-time.
 There are many ways a search index could be built and maintained, but Backstage
 Search chooses to completely rebuild indices on a schedule. Different collators
 can be configured to refresh at different intervals, depending on how often the
-source information is updated.
+source information is updated. When search indexing is distributed among multiple
+backend nodes, coordination to prevent clashes is typically handled by a
+distributed `TaskRunner`.
 
 ### The Search Page
 

docs/features/search/getting-started.md

@@ -149,6 +149,7 @@ import {
 import { PluginEnvironment } from '../types';
 import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
 import { Router } from 'express';
+import { Duration } from 'luxon';
 
 export default async function createPlugin(
   env: PluginEnvironment,
@@ -161,9 +162,15 @@ export default async function createPlugin(
     searchEngine,
   });
 
+  const every10MinutesSchedule = env.scheduler.createScheduledTaskRunner({
+    frequency: Duration.fromObject({ seconds: 600 }),
+    timeout: Duration.fromObject({ seconds: 900 }),
+    initialDelay: Duration.fromObject({ seconds: 3 }),
+  });
+
   indexBuilder.addCollator({
-    defaultRefreshIntervalSeconds: 600,
-    collator: new DefaultCatalogCollator({
+    schedule: every10MinutesSchedule,
+    factory: DefaultCatalogCollatorFactory.fromConfig(env.config, {
       discovery: env.discovery,
       tokenManager: env.tokenManager,
     }),
@@ -287,32 +294,87 @@ which are responsible for providing documents
 number of collators with the `IndexBuilder` like this:
 
 ```typescript
+import { Duration } from 'luxon';
+
 const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine });
 
+const every10MinutesSchedule = env.scheduler.createScheduledTaskRunner({
+  frequency: Duration.fromObject({ seconds: 600 }),
+  timeout: Duration.fromObject({ seconds: 900 }),
+  initialDelay: Duration.fromObject({ seconds: 3 }),
+});
+
+const everyHourSchedule = env.scheduler.createScheduledTaskRunner({
+  frequency: Duration.fromObject({ seconds: 3600 }),
+  timeout: Duration.fromObject({ seconds: 5400 }),
+  initialDelay: Duration.fromObject({ seconds: 3 }),
+});
+
 indexBuilder.addCollator({
-  defaultRefreshIntervalSeconds: 600,
-  collator: new DefaultCatalogCollator({
+  schedule: every10MinutesSchedule,
+  factory: DefaultCatalogCollatorFactory.fromConfig(env.config, {
     discovery: env.discovery,
     tokenManager: env.tokenManager,
   }),
 });
 
 indexBuilder.addCollator({
-  defaultRefreshIntervalSeconds: 3600,
-  collator: new MyCustomCollator(),
+  schedule: everyHourSchedule,
+  factory: new MyCustomCollatorFactory(),
 });
 ```
 
 Backstage Search builds and maintains its index
 [on a schedule](./concepts.md#the-scheduler). You can change how often the
 indexes are rebuilt for a given type of document. You may want to do this if
-your documents are updated more or less frequently. You can do so by modifying
-its `defaultRefreshIntervalSeconds` value, like this:
+your documents are updated more or less frequently. You can do so by configuring
+a scheduled `TaskRunner` to pass into the `schedule` value, like this:
 
 ```typescript {3}
+const every10MinutesSchedule = env.scheduler.createScheduledTaskRunner({
+  frequency: Duration.fromObject({ seconds: 600 }),
+  timeout: Duration.fromObject({ seconds: 900 }),
+  initialDelay: Duration.fromObject({ seconds: 3 }),
+});
+
 indexBuilder.addCollator({
-  defaultRefreshIntervalSeconds: 600,
-  collator: new DefaultCatalogCollator({
+  schedule: every10MinutesSchedule,
+  factory: DefaultCatalogCollatorFactory.fromConfig(env.config, {
+    discovery: env.discovery,
+    tokenManager: env.tokenManager,
+  }),
+});
+```
+
+Note: if you are using the in-memory Lunr search engine, you probably want to
+implement a non-distributed `TaskRunner` like the following to ensure consistency
+if you're running multiple search backend nodes (alternatively, you can configure
+the search plugin to use a non-distributed database such as
+[SQLite](../../tutorials/configuring-plugin-databases.md#postgresql-and-sqlite-3)):
+
+```typescript
+import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
+
+const schedule: TaskRunner = {
+  run: async (task: TaskInvocationDefinition) => {
+    const startRefresh = async () => {
+      while (!task.signal?.aborted) {
+        try {
+          await task.fn(task.signal);
+        } catch {
+          // ignore intentionally
+        }
+
+        await new Promise(resolve => setTimeout(resolve, 600 * 1000));
+      }
+    };
+    startRefresh();
+  },
+};
+
+indexBuilder.addCollator({
+  schedule,
+  factory: DefaultCatalogCollatorFactory.fromConfig(env.config, {
     discovery: env.discovery,
     tokenManager: env.tokenManager,
   }),