Update core search types to be stream-based

Signed-off-by: Eric Peterson <ericpeterson@spotify.com>

packages/search-common/api-report.md

@@ -3,25 +3,28 @@
 > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
 
 ```ts
+/// <reference types="node" />
+
 import { JsonObject } from '@backstage/types';
 import { Permission } from '@backstage/plugin-permission-common';
+import { Readable } from 'stream';
+import { Transform } from 'stream';
+import { Writable } from 'stream';
 
-// Warning: (ae-missing-release-tag) "DocumentCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+// Warning: (ae-missing-release-tag) "DocumentCollatorFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public
-export interface DocumentCollator {
-  // (undocumented)
-  execute(): Promise<IndexableDocument[]>;
+export interface DocumentCollatorFactory {
+  getCollator(): Promise<Readable>;
   readonly type: string;
   readonly visibilityPermission?: Permission;
 }
 
-// Warning: (ae-missing-release-tag) "DocumentDecorator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+// Warning: (ae-missing-release-tag) "DocumentDecoratorFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public
-export interface DocumentDecorator {
-  // (undocumented)
-  execute(documents: IndexableDocument[]): Promise<IndexableDocument[]>;
+export interface DocumentDecoratorFactory {
+  getDecorator(): Promise<Transform>;
   readonly types?: string[];
 }
 
@@ -60,7 +63,7 @@ export type QueryTranslator = (query: SearchQuery) => unknown;
 //
 // @public
 export interface SearchEngine {
-  index(type: string, documents: IndexableDocument[]): Promise<void>;
+  getIndexer(type: string): Promise<Writable>;
   query(
     query: SearchQuery,
     options?: QueryRequestOptions,

packages/search-common/src/types.ts

@@ -16,6 +16,7 @@
 
 import { Permission } from '@backstage/plugin-permission-common';
 import { JsonObject } from '@backstage/types';
+import { Readable, Transform, Writable } from 'stream';
 
 export interface SearchQuery {
   term: string;
@@ -82,10 +83,9 @@ export type DocumentTypeInfo = {
 };
 
 /**
- * Interface that must be implemented in order to expose new documents to
- * search.
+ * Factory class for instantiating collators.
  */
-export interface DocumentCollator {
+export interface DocumentCollatorFactory {
   /**
    * The type or name of the document set returned by this collator. Used as an
    * index name by Search Engines.
@@ -98,21 +98,27 @@ export interface DocumentCollator {
    */
   readonly visibilityPermission?: Permission;
 
-  execute(): Promise<IndexableDocument[]>;
+  /**
+   * Instantiates and resolves a document collator.
+   */
+  getCollator(): Promise<Readable>;
 }
 
 /**
- * Interface that must be implemented in order to decorate existing documents with
- * additional metadata.
+ * Factory class for instantiating decorators.
  */
-export interface DocumentDecorator {
+export interface DocumentDecoratorFactory {
   /**
    * An optional array of document/index types on which this decorator should
    * be applied. If no types are provided, this decorator will be applied to
    * all document/index types.
    */
   readonly types?: string[];
-  execute(documents: IndexableDocument[]): Promise<IndexableDocument[]>;
+
+  /**
+   * Instantiates and resolves a document decorator.
+   */
+  getDecorator(): Promise<Transform>;
 }
 
 /**
@@ -137,9 +143,15 @@ export interface SearchEngine {
   setTranslator(translator: QueryTranslator): void;
 
   /**
-   * Add the given documents to the SearchEngine index of the given type.
+   * Factory method for getting a search engine indexer for a given document
+   * type.
+   *
+   * @param type - The type or name of the document set for which an indexer
+   *   should be retrieved. This corresponds to the `type` property on the
+   *   document collator/decorator factories and will most often be used to
+   *   identify an index or group to which documents should be written.
    */
-  index(type: string, documents: IndexableDocument[]): Promise<void>;
+  getIndexer(type: string): Promise<Writable>;
 
   /**
    * Perform a search query against the SearchEngine.

plugins/search-backend-node/src/types.ts

@@ -14,7 +14,10 @@
  * limitations under the License.
  */
 
-import { DocumentCollator, DocumentDecorator } from '@backstage/search-common';
+import {
+  DocumentCollatorFactory,
+  DocumentDecoratorFactory,
+} from '@backstage/search-common';
 
 /**
  * Parameters required to register a collator.
@@ -26,9 +29,9 @@ export interface RegisterCollatorParameters {
   defaultRefreshIntervalSeconds: number;
 
   /**
-   * The collator class responsible for returning all documents of the given type.
+   * The class responsible for returning the document collator of the given type.
    */
-  collator: DocumentCollator;
+  factory: DocumentCollatorFactory;
 }
 
 /**
@@ -36,7 +39,7 @@ export interface RegisterCollatorParameters {
  */
 export interface RegisterDecoratorParameters {
   /**
-   * The decorator class responsible for appending or modifying documents of the given type(s).
+   * The class responsible for returning the decorator which appends, modifies, or filters documents.
    */
-  decorator: DocumentDecorator;
+  factory: DocumentDecoratorFactory;
 }

plugins/search-backend/src/service/router.test.ts

@@ -105,7 +105,7 @@ describe('createRouter', () => {
       beforeAll(async () => {
         const logger = getVoidLogger();
         mockSearchEngine = {
-          index: jest.fn(),
+          getIndexer: jest.fn(),
           setTranslator: jest.fn(),
           query: jest.fn(),
         };