Merge pull request #9839 from backstage/search/as-streams

packages/backend/src/plugins/search.ts

@@ -18,7 +18,7 @@ import {
   useHotCleanup,
 } from '@backstage/backend-common';
 import { Config } from '@backstage/config';
-import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
+import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend';
 import { createRouter } from '@backstage/plugin-search-backend';
 import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch';
 import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg';
@@ -27,7 +27,7 @@ import {
   LunrSearchEngine,
   SearchEngine,
 } from '@backstage/plugin-search-backend-node';
-import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
+import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
 import { Logger } from 'winston';
 import { PluginEnvironment } from '../types';
 
@@ -70,7 +70,7 @@ export default async function createPlugin({
   // particular collator gathers entities from the software catalog.
   indexBuilder.addCollator({
     defaultRefreshIntervalSeconds: 600,
-    collator: DefaultCatalogCollator.fromConfig(config, {
+    factory: DefaultCatalogCollatorFactory.fromConfig(config, {
       discovery,
       tokenManager,
     }),
@@ -78,7 +78,7 @@ export default async function createPlugin({
 
   indexBuilder.addCollator({
     defaultRefreshIntervalSeconds: 600,
-    collator: DefaultTechDocsCollator.fromConfig(config, {
+    factory: DefaultTechDocsCollatorFactory.fromConfig(config, {
       discovery,
       logger,
       tokenManager,

packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts

@@ -5,8 +5,8 @@ import {
   LunrSearchEngine,
 } from '@backstage/plugin-search-backend-node';
 import { PluginEnvironment } from '../types';
-import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
-import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
+import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend';
+import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
 
 export default async function createPlugin({
   logger,
@@ -23,7 +23,7 @@ export default async function createPlugin({
   // collator gathers entities from the software catalog.
   indexBuilder.addCollator({
     defaultRefreshIntervalSeconds: 600,
-    collator: DefaultCatalogCollator.fromConfig(config, {
+    factory: DefaultCatalogCollatorFactory.fromConfig(config, {
       discovery,
       tokenManager,
     }),
@@ -32,7 +32,7 @@ export default async function createPlugin({
   // collator gathers entities from techdocs.
   indexBuilder.addCollator({
     defaultRefreshIntervalSeconds: 600,
-    collator: DefaultTechDocsCollator.fromConfig(config, {
+    factory: DefaultTechDocsCollatorFactory.fromConfig(config, {
       discovery,
       logger,
       tokenManager,

packages/search-common/api-report.md

@@ -3,38 +3,33 @@
 > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
 
 ```ts
+/// <reference types="node" />
+
 import { JsonObject } from '@backstage/types';
 import { Permission } from '@backstage/plugin-permission-common';
+import { Readable } from 'stream';
+import { Transform } from 'stream';
+import { Writable } from 'stream';
 
-// Warning: (ae-missing-release-tag) "DocumentCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public
-export interface DocumentCollator {
-  // (undocumented)
-  execute(): Promise<IndexableDocument[]>;
+// @beta
+export interface DocumentCollatorFactory {
+  getCollator(): Promise<Readable>;
   readonly type: string;
   readonly visibilityPermission?: Permission;
 }
 
-// Warning: (ae-missing-release-tag) "DocumentDecorator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public
-export interface DocumentDecorator {
-  // (undocumented)
-  execute(documents: IndexableDocument[]): Promise<IndexableDocument[]>;
+// @beta
+export interface DocumentDecoratorFactory {
+  getDecorator(): Promise<Transform>;
   readonly types?: string[];
 }
 
-// Warning: (ae-missing-release-tag) "DocumentTypeInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public
+// @beta
 export type DocumentTypeInfo = {
   visibilityPermission?: Permission;
 };
 
-// Warning: (ae-missing-release-tag) "IndexableDocument" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public
+// @beta
 export interface IndexableDocument {
   authorization?: {
     resourceRef: string;
@@ -44,23 +39,17 @@ export interface IndexableDocument {
   title: string;
 }
 
-// Warning: (ae-missing-release-tag) "QueryRequestOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
+// @beta
 export type QueryRequestOptions = {
   token?: string;
 };
 
-// Warning: (ae-missing-release-tag) "QueryTranslator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public
+// @beta
 export type QueryTranslator = (query: SearchQuery) => unknown;
 
-// Warning: (ae-missing-release-tag) "SearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public
+// @beta
 export interface SearchEngine {
-  index(type: string, documents: IndexableDocument[]): Promise<void>;
+  getIndexer(type: string): Promise<Writable>;
   query(
     query: SearchQuery,
     options?: QueryRequestOptions,
@@ -68,9 +57,7 @@ export interface SearchEngine {
   setTranslator(translator: QueryTranslator): void;
 }
 
-// Warning: (ae-missing-release-tag) "SearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
+// @beta (undocumented)
 export interface SearchQuery {
   // (undocumented)
   filters?: JsonObject;
@@ -82,9 +69,7 @@ export interface SearchQuery {
   types?: string[];
 }
 
-// Warning: (ae-missing-release-tag) "SearchResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
+// @beta (undocumented)
 export interface SearchResult {
   // (undocumented)
   document: IndexableDocument;
@@ -92,9 +77,7 @@ export interface SearchResult {
   type: string;
 }
 
-// Warning: (ae-missing-release-tag) "SearchResultSet" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
+// @beta (undocumented)
 export interface SearchResultSet {
   // (undocumented)
   nextPageCursor?: string;

packages/search-common/src/types.ts

@@ -16,7 +16,11 @@
 
 import { Permission } from '@backstage/plugin-permission-common';
 import { JsonObject } from '@backstage/types';
+import { Readable, Transform, Writable } from 'stream';
 
+/**
+ * @beta
+ */
 export interface SearchQuery {
   term: string;
   filters?: JsonObject;
@@ -24,11 +28,17 @@ export interface SearchQuery {
   pageCursor?: string;
 }
 
+/**
+ * @beta
+ */
 export interface SearchResult {
   type: string;
   document: IndexableDocument;
 }
 
+/**
+ * @beta
+ */
 export interface SearchResultSet {
   results: SearchResult[];
   nextPageCursor?: string;
@@ -38,6 +48,7 @@ export interface SearchResultSet {
 /**
  * Base properties that all indexed documents must include, as well as some
  * common properties that documents are encouraged to use where appropriate.
+ * @beta
  */
 export interface IndexableDocument {
   /**
@@ -72,6 +83,7 @@ export interface IndexableDocument {
  * Information about a specific document type. Intended to be used in the
  * {@link @backstage/search-backend-node#IndexBuilder} to collect information
  * about the types stored in the index.
+ * @beta
  */
 export type DocumentTypeInfo = {
   /**
@@ -82,10 +94,10 @@ export type DocumentTypeInfo = {
 };
 
 /**
- * Interface that must be implemented in order to expose new documents to
- * search.
+ * Factory class for instantiating collators.
+ * @beta
  */
-export interface DocumentCollator {
+export interface DocumentCollatorFactory {
   /**
    * The type or name of the document set returned by this collator. Used as an
    * index name by Search Engines.
@@ -98,29 +110,41 @@ export interface DocumentCollator {
    */
   readonly visibilityPermission?: Permission;
 
-  execute(): Promise<IndexableDocument[]>;
+  /**
+   * Instantiates and resolves a document collator.
+   */
+  getCollator(): Promise<Readable>;
 }
 
 /**
- * Interface that must be implemented in order to decorate existing documents with
- * additional metadata.
+ * Factory class for instantiating decorators.
+ * @beta
  */
-export interface DocumentDecorator {
+export interface DocumentDecoratorFactory {
   /**
    * An optional array of document/index types on which this decorator should
    * be applied. If no types are provided, this decorator will be applied to
    * all document/index types.
    */
   readonly types?: string[];
-  execute(documents: IndexableDocument[]): Promise<IndexableDocument[]>;
+
+  /**
+   * Instantiates and resolves a document decorator.
+   */
+  getDecorator(): Promise<Transform>;
 }
 
 /**
  * A type of function responsible for translating an abstract search query into
  * a concrete query relevant to a particular search engine.
+ * @beta
  */
 export type QueryTranslator = (query: SearchQuery) => unknown;
 
+/**
+ * Options when querying a search engine.
+ * @beta
+ */
 export type QueryRequestOptions = {
   token?: string;
 };
@@ -129,6 +153,7 @@ export type QueryRequestOptions = {
  * Interface that must be implemented by specific search engines, responsible
  * for performing indexing and querying and translating abstract queries into
  * concrete, search engine-specific queries.
+ * @beta
  */
 export interface SearchEngine {
   /**
@@ -137,9 +162,15 @@ export interface SearchEngine {
   setTranslator(translator: QueryTranslator): void;
 
   /**
-   * Add the given documents to the SearchEngine index of the given type.
+   * Factory method for getting a search engine indexer for a given document
+   * type.
+   *
+   * @param type - The type or name of the document set for which an indexer
+   *   should be retrieved. This corresponds to the `type` property on the
+   *   document collator/decorator factories and will most often be used to
+   *   identify an index or group to which documents should be written.
    */
-  index(type: string, documents: IndexableDocument[]): Promise<void>;
+  getIndexer(type: string): Promise<Writable>;
 
   /**
    * Perform a search query against the SearchEngine.