Merge pull request #9770 from kuangp/feat/searchCoordination

feat(search): handle search indexing coordination among nodes

packages/backend/package.json

@@ -68,6 +68,7 @@
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
     "express-prom-bundle": "^6.3.6",
+    "luxon": "^2.0.2",
     "pg": "^8.3.0",
     "pg-connection-string": "^2.3.0",
     "prom-client": "^14.0.1",
@@ -77,7 +78,8 @@
     "@backstage/cli": "^0.17.0-next.1",
     "@types/dockerode": "^3.3.0",
     "@types/express": "^4.17.6",
-    "@types/express-serve-static-core": "^4.17.5"
+    "@types/express-serve-static-core": "^4.17.5",
+    "@types/luxon": "^2.0.4"
   },
   "files": [
     "dist"

packages/backend/src/plugins/search.ts

@@ -26,6 +26,7 @@ import {
 } from '@backstage/plugin-search-backend-node';
 import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
 import { Router } from 'express';
+import { Duration } from 'luxon';
 import { PluginEnvironment } from '../types';
 
 async function createSearchEngine(
@@ -55,10 +56,18 @@ export default async function createPlugin(
     searchEngine,
   });
 
+  const schedule = env.scheduler.createScheduledTaskRunner({
+    frequency: Duration.fromObject({ seconds: 600 }),
+    timeout: Duration.fromObject({ seconds: 900 }),
+    // A 3 second delay gives the backend server a chance to initialize before
+    // any collators are executed, which may attempt requests against the API.
+    initialDelay: Duration.fromObject({ seconds: 3 }),
+  });
+
   // Collators are responsible for gathering documents known to plugins. This
   // particular collator gathers entities from the software catalog.
   indexBuilder.addCollator({
-    defaultRefreshIntervalSeconds: 600,
+    schedule,
     factory: DefaultCatalogCollatorFactory.fromConfig(env.config, {
       discovery: env.discovery,
       tokenManager: env.tokenManager,
@@ -66,7 +75,7 @@ export default async function createPlugin(
   });
 
   indexBuilder.addCollator({
-    defaultRefreshIntervalSeconds: 600,
+    schedule,
     factory: DefaultTechDocsCollatorFactory.fromConfig(env.config, {
       discovery: env.discovery,
       logger: env.logger,
@@ -77,10 +86,8 @@ export default async function createPlugin(
   // The scheduler controls when documents are gathered from collators and sent
   // to the search engine for indexing.
   const { scheduler } = await indexBuilder.build();
+  scheduler.start();
 
-  // A 3 second delay gives the backend server a chance to initialize before
-  // any collators are executed, which may attempt requests against the API.
-  setTimeout(() => scheduler.start(), 3000);
   useHotCleanup(module, () => scheduler.stop());
 
   return await createRouter({

packages/create-app/templates/default-app/packages/backend/package.json.hbs

@@ -40,6 +40,7 @@
     "dockerode": "^3.3.1",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
+    "luxon": "^2.0.2",
     {{#if dbTypePG}}
     "pg": "^8.3.0",
     {{/if}}
@@ -52,7 +53,8 @@
     "@backstage/cli": "^{{version '@backstage/cli'}}",
     "@types/dockerode": "^3.3.0",
     "@types/express": "^4.17.6",
-    "@types/express-serve-static-core": "^4.17.5"
+    "@types/express-serve-static-core": "^4.17.5",
+    "@types/luxon": "^2.0.4"
   },
   "files": [
     "dist"

packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs

@@ -11,6 +11,7 @@ import { PluginEnvironment } from '../types';
 import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend';
 import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
 import { Router } from 'express';
+import { Duration } from 'luxon';
 
 export default async function createPlugin(
   env: PluginEnvironment,
@@ -31,10 +32,18 @@ export default async function createPlugin(
     searchEngine,
   });
 
+  const schedule = env.scheduler.createScheduledTaskRunner({
+    frequency: Duration.fromObject({ seconds: 600 }),
+    timeout: Duration.fromObject({ seconds: 900 }),
+    // A 3 second delay gives the backend server a chance to initialize before
+    // any collators are executed, which may attempt requests against the API.
+    initialDelay: Duration.fromObject({ seconds: 3 }),
+  });
+
   // Collators are responsible for gathering documents known to plugins. This
   // collator gathers entities from the software catalog.
   indexBuilder.addCollator({
-    defaultRefreshIntervalSeconds: 600,
+    schedule,
     factory: DefaultCatalogCollatorFactory.fromConfig(env.config, {
       discovery: env.discovery,
       tokenManager: env.tokenManager,
@@ -43,7 +52,7 @@ export default async function createPlugin(
 
   // collator gathers entities from techdocs.
   indexBuilder.addCollator({
-    defaultRefreshIntervalSeconds: 600,
+    schedule,
     factory: DefaultTechDocsCollatorFactory.fromConfig(env.config, {
       discovery: env.discovery,
       logger: env.logger,
@@ -54,10 +63,8 @@ export default async function createPlugin(
   // The scheduler controls when documents are gathered from collators and sent
   // to the search engine for indexing.
   const { scheduler } = await indexBuilder.build();
+  scheduler.start();
 
-  // A 3 second delay gives the backend server a chance to initialize before
-  // any collators are executed, which may attempt requests against the API.
-  setTimeout(() => scheduler.start(), 3000);
   useHotCleanup(module, () => scheduler.stop());
 
   return await createRouter({