From 1d8d3c408ad72621166880d9882a7b39ee606fa0 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 8 Apr 2022 18:13:05 +0200 Subject: [PATCH 1/2] Use human-readable durations throughout. Signed-off-by: Eric Peterson --- .changeset/ninety-eggs-argue.md | 4 ++-- docs/features/search/getting-started.md | 16 ++++++++-------- packages/backend/src/plugins/search.ts | 4 ++-- .../packages/backend/src/plugins/search.ts.hbs | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.changeset/ninety-eggs-argue.md b/.changeset/ninety-eggs-argue.md index 01f861fe4b..6fd221b722 100644 --- a/.changeset/ninety-eggs-argue.md +++ b/.changeset/ninety-eggs-argue.md @@ -17,8 +17,8 @@ To make this change to an existing app, make the following changes to `packages/ /* ... */ + const schedule = env.scheduler.createScheduledTaskRunner({ -+ frequency: Duration.fromObject({ seconds: 600 }), -+ timeout: Duration.fromObject({ seconds: 900 }), ++ frequency: Duration.fromObject({ minutes: 10 }), ++ timeout: Duration.fromObject({ minutes: 15 }), + initialDelay: Duration.fromObject({ seconds: 3 }), + }); diff --git a/docs/features/search/getting-started.md b/docs/features/search/getting-started.md index bc17b1eea2..1557a29b53 100644 --- a/docs/features/search/getting-started.md +++ b/docs/features/search/getting-started.md @@ -163,8 +163,8 @@ export default async function createPlugin( }); const every10MinutesSchedule = env.scheduler.createScheduledTaskRunner({ - frequency: Duration.fromObject({ seconds: 600 }), - timeout: Duration.fromObject({ seconds: 900 }), + frequency: Duration.fromObject({ minutes: 10 }), + timeout: Duration.fromObject({ minutes: 15 }), initialDelay: Duration.fromObject({ seconds: 3 }), }); @@ -299,14 +299,14 @@ import { Duration } from 'luxon'; const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine }); const every10MinutesSchedule = env.scheduler.createScheduledTaskRunner({ - frequency: Duration.fromObject({ seconds: 600 }), - timeout: Duration.fromObject({ seconds: 900 }), + frequency: Duration.fromObject({ minutes: 10 }), + timeout: Duration.fromObject({ minutes: 15 }), initialDelay: Duration.fromObject({ seconds: 3 }), }); const everyHourSchedule = env.scheduler.createScheduledTaskRunner({ - frequency: Duration.fromObject({ seconds: 3600 }), - timeout: Duration.fromObject({ seconds: 5400 }), + frequency: Duration.fromObject({ hours: 1 }), + timeout: Duration.fromObject({ minutes: 90 }), initialDelay: Duration.fromObject({ seconds: 3 }), }); @@ -332,8 +332,8 @@ a scheduled `TaskRunner` to pass into the `schedule` value, like this: ```typescript {3} const every10MinutesSchedule = env.scheduler.createScheduledTaskRunner({ - frequency: Duration.fromObject({ seconds: 600 }), - timeout: Duration.fromObject({ seconds: 900 }), + frequency: Duration.fromObject({ minutes: 10 }), + timeout: Duration.fromObject({ minutes: 15 }), initialDelay: Duration.fromObject({ seconds: 3 }), }); diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 052b7380c3..fc91d98cfe 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -57,8 +57,8 @@ export default async function createPlugin( }); const schedule = env.scheduler.createScheduledTaskRunner({ - frequency: Duration.fromObject({ seconds: 600 }), - timeout: Duration.fromObject({ seconds: 900 }), + frequency: Duration.fromObject({ minutes: 10 }), + timeout: Duration.fromObject({ minutes: 15 }), // A 3 second delay gives the backend server a chance to initialize before // any collators are executed, which may attempt requests against the API. initialDelay: Duration.fromObject({ seconds: 3 }), diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs index bda10ae21b..62bc3056cc 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs @@ -33,8 +33,8 @@ export default async function createPlugin( }); const schedule = env.scheduler.createScheduledTaskRunner({ - frequency: Duration.fromObject({ seconds: 600 }), - timeout: Duration.fromObject({ seconds: 900 }), + frequency: Duration.fromObject({ minutes: 10 }), + timeout: Duration.fromObject({ minutes: 15 }), // A 3 second delay gives the backend server a chance to initialize before // any collators are executed, which may attempt requests against the API. initialDelay: Duration.fromObject({ seconds: 3 }), From 5960b78a4cd175581827c374778706581a9ea5fc Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 8 Apr 2022 18:13:45 +0200 Subject: [PATCH 2/2] Document task scheduler / DB. And improve throughout. Signed-off-by: Eric Peterson --- docs/assets/search/architecture.drawio.svg | 945 ++++++++++++++------- 1 file changed, 637 insertions(+), 308 deletions(-) diff --git a/docs/assets/search/architecture.drawio.svg b/docs/assets/search/architecture.drawio.svg index d7a8177d1c..709a008b43 100644 --- a/docs/assets/search/architecture.drawio.svg +++ b/docs/assets/search/architecture.drawio.svg @@ -1,20 +1,19 @@ - + - - - - - - - - + + + + + + + - -
-
-
+ +
+
+

@@ -22,18 +21,17 @@
- - + - +
-
-
+
+

@@ -42,20 +40,38 @@
- - - + + + + + +
+
+
+
+ Search +
+ Engine +
+
+
+ + + Search... + + + - +
-
-
+
+
App Package: <Route path="/search" element={<... />} />
@@ -66,51 +82,51 @@ - + - -
-
-
- @backstage/ -
- plugin-search-backend + +
+
+
+ + @backstage/ +
+ plugin-search-backend +
- + @backstage/... - + - -
-
-
- Other Plugins -
- (TechDocs, Catalog, Etc) + +
+
+
+ @backstage/plugin-xyz
- - Other Plugins... + + @backstage/plugin-xyz - + - -
-
-
+ +
+
+

@@ -120,94 +136,90 @@
- + - + - -
-
-
+ +
+
+
@backstage/plugin-search
- + @backstage/plugin-search - - - + - -
-
-
- Other Backend Plugin (TechDocs, Catalog, Etc) + +
+
+
+ @backstage/plugin-xyz-backend
- - Other Backend Plugin (TechDocs, Catalog, Etc) + + @backstage/plugin-xyz-backend - + - -
-
-
- Search Engine (Elastic, Solr, SaaS, etc.) + +
+
+
+ e.g. ElasticSearch, Postgres, Lunr, etc.
- - Search Engine (Elastic, Solr, SaaS, et... + + e.g. ElasticSearch, Postgres, Lunr,... - + - -
-
-
- Search Engine Integration Layer + +
+
+
+ @backstage/plugin-search-backend-module-xyz
- - Search Engine Integration Layer + + @backstage/plugin-search-backend-module-xyz - - - - - + + + - +
-
-
+
+
1 2 3 @@ -223,10 +235,10 @@ - +
-
-
+
+
X number of search results @@ -239,17 +251,17 @@ - - - - - + + + + + - -
-
-
+ +
+
+
Components @@ -257,43 +269,18 @@
- + Components - - - + - -
-
-
- Pass Search -
- Term and Filters -
- and then -
- Return Results -
-
-
- - - Pass Search... - - - - - - - -
-
-
+ +
+
+
Search API @@ -301,20 +288,20 @@
- + Search API - - + + - -
-
-
+ +
+
+
Components @@ -322,232 +309,574 @@
- + Components - - + + - -
-
-
+ +
+
+
+ + IndexBuilder + +
+
+
+ + + IndexBui... + + + + + + + + + + +
+
+
+ + Query Service +
+ +
+
+
+ + + Query Se... + + + + + + + +
+
+
+ + + Indexer + +
+
+ Manages Indices +
+ and Writes Documents to a Search Engine +
+ +
+
+
+ + + Indexer... + + + + + + + + + + +
+
+
+ + + Query Handler + +
+
+ Compiles and Executes Query Against a Search Engine + +
+
+
+ + + Query Handler... + + + + + + + +
+
+
+ @backstage/ +
+ plugin-search-backend-node + +
+
+
+ + + @backstage/... + + + + + + + +
+
+
+ Backend Package: src/plugins/search.ts +
+
+
+ + + Backend Package: src/plugins/search.ts + + + + + + + +
+
+
+
+ +
+ +
+
+
+
+ + + + + + + + +
+
+
+ + + Authorization +
+ (Optional) + +
+ +
+
+
+ + + Authorization... + + + + + + + + + + +
+
+
+ IndexBuilder +
+
+
+ + + IndexBuilder + + + + + + + +
+
+
+ + SearchEngine + +
+
+
+ + + SearchEngine + + + + + + + +
+
+
+ + Collator(s) + +
+
+
+ + + Collator(s) + + + + + + + +
+
+
+ + Decorator(s) + +
+
+
+ + + Decorator(s) + + + + + + + +
+
+
+ + Start Schedule + +
+
+
+ + + Start Schedule + + + + + + + +
+
+
+ + Create Router + +
+
+
+ + + Create Router + + + + + + + +
+
+
+ Scheduler
- + Scheduler - + + + + + - -
-
-
- - Gather Documents From Plugins - + +
+
+
+
+ + Database for + +
+ +
+ + Task Coordination + +
+
+ + Among Nodes + +
+
- - Gather D... + + Database fo... - - - - + + - -
-
-
- - Register Document / Metadata Collation Handler(s) - + +
+
+
+
+ + Install and configure the +
+ search engine, collators, +
+ and decorators that are +
+ appropriate for your +
+ organization! + +
- - Register Docu... + + Install and con... - + + - -
-
-
- - API Endpoint + +
+
+
+ + Custom Query
+ Translator (Optional)
- - API Endp... + + Custom Quer... - + - -
-
-
+ +
+
+
+ + SearchEngine Implementation + +
+
+
+ + + SearchEngine Implementati... + + + + + + + +
+
+
+ + XyzCollatorFactory + +
+
+
+ + + XyzCollatorFactory + + + + + + + + + + +
+
+
+ + XyzDecoratorFactory + +
+
+
+ + + XyzDecoratorFacto... + + + + + + + + + + + + + + + + + +
+
+
+
+ + Individual backend plugins +
+ define how documents are +
+ retrieved from the plugin's +
+ data store and mapped to +
+ an IndexableDocument. + +
+
+
+
+ + + Individual backe... + + + + + + + + + + + + + + + + + +
+
+
+
+ + Individual frontend plugins may +
+ define custom components, +
+ e.g. custom search result items. + +
+
+
+
+ + + Individual frontend... + + + + + + + +
+
+
- Query Processing + Search Context
- - Query Processing - - - - - - - -
-
-
- - Index Processing - -
-
-
- - - Index Processi... - - - - - - - -
-
-
- - Index Processing - -
-
-
- - - Index Processi... - - - - - - - -
-
-
- - Query Processing - -
-
-
- - - Query Processing - - - - - - - -
-
-
- - - Manage Index - -
- Create, Remove, Replace Documents and Indices - -
-
-
- - - Manage Index... - - - - - - - - - - -
-
-
- - Compile and Execute Query from Term and Filters - -
-
-
- - - Compile and Execute... - - - - - - - -
-
-
- @backstage/ -
- plugin-search-backend-node -
-
-
- - - @backstage/... + + Search Cont... + + + + + + + + +