Merge pull request #9839 from backstage/search/as-streams
This commit is contained in:
Eric Peterson
@@ -0,0 +1,18 @@
|
||||
---
|
||||
'@backstage/plugin-search-backend-node': minor
|
||||
'@backstage/search-common': minor
|
||||
---
|
||||
|
||||
**BREAKING**
|
||||
|
||||
The Backstage Search Platform's indexing process has been rewritten as a stream
|
||||
pipeline in order to improve efficiency and performance on large document sets.
|
||||
|
||||
The concepts of `Collator` and `Decorator` have been replaced with readable and
|
||||
transform object streams (respectively), as well as factory classes to
|
||||
instantiate them. Accordingly, the `SearchEngine.index()` method has also been
|
||||
replaced with a `getIndexer()` factory method that resolves to a writable
|
||||
object stream.
|
||||
|
||||
Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta)
|
||||
for further details.
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
'@backstage/plugin-search-backend-module-pg': minor
|
||||
---
|
||||
|
||||
**BREAKING**
|
||||
|
||||
The `PgSearchEngine` implements the new stream-based indexing process expected
|
||||
by the latest `@backstage/search-backend-node`.
|
||||
|
||||
When updating to this version, you must also update to the latest version of
|
||||
`@backstage/search-backend-node`. Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta)
|
||||
for further details.
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
'@backstage/plugin-techdocs-backend': patch
|
||||
---
|
||||
|
||||
A `DefaultTechDocsCollatorFactory`, which works with the new stream-based
|
||||
search indexing subsystem, is now available. The `DefaultTechDocsCollator` will
|
||||
continue to be available for those unable to upgrade to the stream-based
|
||||
`@backstage/search-backend-node` (and related packages), however it is now
|
||||
marked as deprecated and will be removed in a future version.
|
||||
|
||||
To upgrade this plugin and the search indexing subsystem in one go, check
|
||||
[this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta)
|
||||
for necessary changes to your search backend plugin configuration.
|
||||
@@ -0,0 +1,43 @@
|
||||
---
|
||||
'@backstage/create-app': patch
|
||||
---
|
||||
|
||||
The Backstage Search Platform's indexing process has been rewritten as a stream
|
||||
pipeline in order to improve efficiency and performance on large document sets.
|
||||
|
||||
To take advantage of this, upgrade to the latest version of
|
||||
`@backstage/plugin-search-backend-node`, as well as any backend plugins whose
|
||||
collators you are using. Then, make the following changes to your
|
||||
`/packages/backend/src/plugins/search.ts` file:
|
||||
|
||||
```diff
|
||||
-import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
|
||||
-import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
|
||||
+import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend';
|
||||
+import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
|
||||
|
||||
// ...
|
||||
|
||||
const indexBuilder = new IndexBuilder({ logger, searchEngine });
|
||||
|
||||
indexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 600,
|
||||
- collator: DefaultCatalogCollator.fromConfig(config, { discovery }),
|
||||
+ factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery }),
|
||||
});
|
||||
|
||||
indexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 600,
|
||||
- collator: DefaultTechDocsCollator.fromConfig(config, {
|
||||
+ factory: DefaultTechDocsCollatorFactory.fromConfig(config, {
|
||||
discovery,
|
||||
logger,
|
||||
}),
|
||||
});
|
||||
```
|
||||
|
||||
If you've written custom collators, decorators, or search engines in your
|
||||
Backstage backend instance, you will need to re-implement them as readable,
|
||||
transform, and writable streams respectively (including factory classes for
|
||||
instantiating them). [A how-to guide for refactoring](https://backstage.io/docs/features/search/how-to-guides#rewriting-alpha-style-collators-for-beta)
|
||||
existing implementations is available.
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
'@backstage/plugin-search-backend-module-elasticsearch': minor
|
||||
---
|
||||
|
||||
**BREAKING**
|
||||
|
||||
The `ElasticSearchSearchEngine` implements the new stream-based indexing
|
||||
process expected by the latest `@backstage/search-backend-node`.
|
||||
|
||||
When updating to this version, you must also update to the latest version of
|
||||
`@backstage/search-backend-node`. Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta)
|
||||
for further details.
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend': patch
|
||||
---
|
||||
|
||||
A `DefaultCatalogCollatorFactory`, which works with the new stream-based
|
||||
search indexing subsystem, is now available. The `DefaultCatalogCollator` will
|
||||
continue to be available for those unable to upgrade to the stream-based
|
||||
`@backstage/search-backend-node` (and related packages), however it is now
|
||||
marked as deprecated and will be removed in a future version.
|
||||
|
||||
To upgrade this plugin and the search indexing subsystem in one go, check
|
||||
[this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta)
|
||||
for necessary changes to your search backend plugin configuration.
|
||||
@@ -213,6 +213,7 @@ parallelization
|
||||
Patrik
|
||||
Peloton
|
||||
performant
|
||||
Performant
|
||||
plantuml
|
||||
Platformize
|
||||
Podman
|
||||
|
||||
@@ -54,13 +54,14 @@ An index is a collection of such documents of a given type.
|
||||
### Collators
|
||||
|
||||
You need to be able to search something! Collators are the way to define what
|
||||
can be searched. Specifically, they're classes which return documents conforming
|
||||
to a minimum set of fields (including a document title, location, and text), but
|
||||
which can contain any other fields as defined by the collator itself. One
|
||||
collator is responsible for defining and collecting documents of a type.
|
||||
can be searched. Specifically, they're readable object streams of documents that
|
||||
conform to a minimum set of fields (including a document title, location, and
|
||||
text), but which can contain any other fields as defined by the collator itself.
|
||||
One collator is responsible for defining and collecting documents of a type.
|
||||
|
||||
Some plugins, like the Catalog Backend, provide so-called "default" collators
|
||||
which you can use out-of-the-box to start searching across Backstage quickly.
|
||||
Some plugins, like the Catalog Backend, provide so-called "default" collator
|
||||
factories which you can use out-of-the-box to start searching across Backstage
|
||||
quickly.
|
||||
|
||||
### Decorators
|
||||
|
||||
@@ -68,9 +69,15 @@ Sometimes you want to add extra information to a set of documents in your search
|
||||
index that the collator may not be aware of. For example, the Software Catalog
|
||||
knows about software entities, but it may not know about their usage or quality.
|
||||
|
||||
Decorators are classes which can add extra fields to pre-collated documents.
|
||||
This extra metadata could then be used to bias search results or otherwise
|
||||
improve the search experience in your Backstage instance.
|
||||
Decorators are transform streams which sit between a collator (read stream) and
|
||||
an indexer (write stream) during the indexing process. It can be used to add
|
||||
extra fields to documents as they are being collated and indexed. This extra
|
||||
metadata could then be used to bias search results or otherwise improve the
|
||||
search experience in your Backstage instance.
|
||||
|
||||
In addition to adding extra metadata, decorators (like any transform stream) can
|
||||
also be used to remove metadata, filter out, or even add extra documents at
|
||||
index-time.
|
||||
|
||||
### The Scheduler
|
||||
|
||||
|
||||
@@ -48,10 +48,10 @@ const app = createApp({
|
||||
## How to index TechDocs documents
|
||||
|
||||
The TechDocs plugin has supported integrations to Search, meaning that it
|
||||
provides a default collator ready to be used.
|
||||
provides a default collator factory ready to be used.
|
||||
|
||||
The purpose of this guide is to walk you through how to register the
|
||||
[DefaultTechDocsCollator](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts)
|
||||
[DefaultTechDocsCollatorFactory](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts)
|
||||
in your App, so that you can get TechDocs documents indexed.
|
||||
|
||||
If you have been through the
|
||||
@@ -60,18 +60,19 @@ you should have the `packages/backend/src/plugins/search.ts` file available. If
|
||||
so, you can go ahead and follow this guide - if not, start by going through the
|
||||
getting started guide.
|
||||
|
||||
1. Import the DefaultTechDocsCollator from `@backstage/plugin-techdocs-backend`.
|
||||
1. Import the `DefaultTechDocsCollatorFactory` from
|
||||
`@backstage/plugin-techdocs-backend`.
|
||||
|
||||
```typescript
|
||||
import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
|
||||
import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
|
||||
```
|
||||
|
||||
2. Register the DefaultTechDocsCollator with the IndexBuilder.
|
||||
2. Register the `DefaultTechDocsCollatorFactory` with the IndexBuilder.
|
||||
|
||||
```typescript
|
||||
indexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 600,
|
||||
collator: DefaultTechDocsCollator.fromConfig(config, {
|
||||
factory: DefaultTechDocsCollatorFactory.fromConfig(config, {
|
||||
discovery,
|
||||
logger,
|
||||
tokenManager,
|
||||
@@ -131,3 +132,264 @@ indexBuilder.addCollator({
|
||||
|
||||
As shown above, you can add a catalog entity filter to narrow down what catalog
|
||||
entities are indexed by the search engine.
|
||||
|
||||
## How to migrate from Search Alpha to Beta
|
||||
|
||||
For the purposes of this guide, Search Beta version is defined as:
|
||||
|
||||
- **Search Plugin**: At least `v0.x.y`
|
||||
- **Search Backend Plugin**: At least `v0.x.y`
|
||||
- **Search Backend Node**: At least `v0.x.y`
|
||||
|
||||
In the Beta version, the Search Platform's indexing process has been rewritten
|
||||
as a stream pipeline in order to improve efficiency and performance on large
|
||||
sets of documents.
|
||||
|
||||
If you've not yet extended the Search Platform with custom code, and have
|
||||
instead taken advantage of default collators, decorators, and search engines
|
||||
provided by existing plugins, the migration process is fairly straightforward:
|
||||
|
||||
1. Upgrade to at least version `0.x.y` of
|
||||
`@backstage/plugin-search-backend-node`, as well as any backend plugins whose
|
||||
collators you are using (e.g. at least version `0.x.y` of
|
||||
`@backstage/plugin-catalog-backend` and/or version `0.x.y` of
|
||||
`@backstage/plugin-techdocs-backend`).
|
||||
2. Then, make the following changes to your
|
||||
`/packages/backend/src/plugins/search.ts` file:
|
||||
|
||||
```diff
|
||||
-import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
|
||||
-import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
|
||||
+import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend';
|
||||
+import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
|
||||
// ...
|
||||
const indexBuilder = new IndexBuilder({ logger, searchEngine });
|
||||
indexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 600,
|
||||
- collator: DefaultCatalogCollator.fromConfig(config, { discovery }),
|
||||
+ factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery }),
|
||||
});
|
||||
indexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 600,
|
||||
- collator: DefaultTechDocsCollator.fromConfig(config, {
|
||||
+ factory: DefaultTechDocsCollatorFactory.fromConfig(config, {
|
||||
discovery,
|
||||
logger,
|
||||
}),
|
||||
});
|
||||
```
|
||||
|
||||
Any custom collators, decorators, or search engine implementations will require
|
||||
minor refactoring. Continue on for details.
|
||||
|
||||
### Rewriting alpha-style collators for beta
|
||||
|
||||
In alpha versions of the Backstage Search Platform, collators were classes that
|
||||
implemented an `execute` method which resolved an `IndexableDocument` array.
|
||||
|
||||
In beta versions, the logic encapsulated by the aforementioned `execute` method
|
||||
is contained within an [object-mode][obj-mode] `Readable` stream where each
|
||||
object pushed onto the stream is of type `IndexableDocument`. Instances of this
|
||||
stream are instantiated by a factory class conforming to the
|
||||
`DocumentCollatorFactory` interface.
|
||||
|
||||
The optimal conversion strategy will vary depending on the collator's logic, but
|
||||
the simplest conversion can follow a process like this:
|
||||
|
||||
1. Rename your collator class to something like `YourCollatorFactory` and update
|
||||
it to implement `DocumentCollatorFactory` instead of `DocumentCollator`.
|
||||
2. Update its `execute` method so that it resolves
|
||||
`AsyncGenerator<YourIndexableDocument>` instead of `YourIndexableDocument[]`.
|
||||
3. Implement `DocumentCollatorFactory`'s `getCollator` method which resolves to
|
||||
`Readable.from(this.execute())` (which is a utility for creating [readable
|
||||
streams][read-stream] from [async generators][async-gen]).
|
||||
|
||||
```ts
|
||||
import { DocumentCollatorFactory } from '@backstage/plugin-search-backend-node';
|
||||
import { Readable } from 'stream';
|
||||
export class YourCollatorFactory implements DocumentCollatorFactory {
|
||||
public readonly type: string = 'your-type';
|
||||
async *execute(): AsyncGenerator<YourIndexableDocument> {
|
||||
const widgets = await this.client.getWidgets();
|
||||
for (const widget of widgets) {
|
||||
yield {
|
||||
title: widget.name,
|
||||
location: widget.url,
|
||||
text: widget.description,
|
||||
};
|
||||
}
|
||||
}
|
||||
getCollator() {
|
||||
return Readable.from(this.execute());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Note: it may be possible to simplify your collator dramatically! If your custom
|
||||
collator was previously using streams under the hood (for example, by reading
|
||||
newline delimited JSON from a local or remote file), you could just expose the
|
||||
stream directly via a simple factory class:
|
||||
|
||||
```ts
|
||||
import { DocumentCollatorFactory } from '@backstage/plugin-search-backend-node';
|
||||
import { createReadStream } from 'fs';
|
||||
import { parse } from '@jsonlines/core';
|
||||
export class YourCollatorFactory implements DocumentCollatorFactory {
|
||||
public readonly type: string = 'your-type';
|
||||
async getCollator() {
|
||||
const parseStream = parse();
|
||||
return createReadStream('./documents.ndjson').pipe(parseStream);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Rewriting alpha-style decorators for beta
|
||||
|
||||
In alpha versions of the Backstage Search Platform, decorators were classes that
|
||||
implemented an `execute` method which took an `IndexableDocument` array as an
|
||||
argument, and resolved a modified array of the same type.
|
||||
|
||||
In beta versions, the logic encapsulated by the aforementioned `execute` method
|
||||
is contained within an object-mode `Transform` stream which reads objects of
|
||||
type `IndexableDocument`, and writes objects of a conforming type. Similar to
|
||||
collators, instances of this stream are instantiated by a factory class
|
||||
conforming to the `DocumentDecoratorFactory` interface.
|
||||
|
||||
Although you can choose to implement a `Transform` stream from scratch, the
|
||||
`@backstage/plugin-search-backend-node` package provides a `DecoratorBase` class
|
||||
in order to simplify the developer experience. With this base class, all that's
|
||||
needed is to transfer your old decorator class logic into the base class' three
|
||||
methods (`initialize`, `decorate`, and `finalize`), and implement the factory
|
||||
class that instantiates the stream:
|
||||
|
||||
```ts
|
||||
import { DecoratorBase } from '@backstage/plugin-search-backend-node';
|
||||
export class YourDecorator extends DecoratorBase {
|
||||
async initialize() {
|
||||
// Setup logic. Performed once before any documents are consumed.
|
||||
}
|
||||
async decorate(
|
||||
document: YourIndexableDocument,
|
||||
): Promise<YourIndexableDocument | YourIndexableDocument[] | undefined> {
|
||||
// Perform transformation logic here.
|
||||
return document;
|
||||
}
|
||||
async finalize() {
|
||||
// Teardown logic. Performed once after all documents have been consumed.
|
||||
}
|
||||
}
|
||||
export class YourDecoratorFactory implements DocumentDecoratorFactory {
|
||||
async getDecorator() {
|
||||
return new YourDecorator();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Note the return type of the `decorate` method and how each can be used to
|
||||
different effect.
|
||||
|
||||
- By resolving a single `YourIndexableDocument` object, your decorator can be
|
||||
used to make simple transformations:
|
||||
|
||||
```ts
|
||||
class BooleanWidgetCoolnessDecorator extends DecoratorBase {
|
||||
async decorator(widget) {
|
||||
// Perform a simple, 1:1 transformation.
|
||||
widget.isCool = widget.isCool === 'true' ? true : false;
|
||||
return widget;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- By resolving `undefined`, your decorator can filter out documents which
|
||||
shouldn't be in the index:
|
||||
|
||||
```ts
|
||||
class OnlyCoolWidgetsDecorator extends DecoratorBase {
|
||||
async decorator(widget) {
|
||||
// Perform a simple filter operation.
|
||||
return widget.isCool ? widget : undefined;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- By resolving an array of `YourIndexableDocument` objects, you can generate
|
||||
multiple documents based on the content of one:
|
||||
|
||||
```ts
|
||||
class WidgetByVariantDecorator extends DecoratorBase {
|
||||
async decorator(widget) {
|
||||
// Generate one widget doc per widget variant.
|
||||
return widget.variants.map(variant => {
|
||||
// Each widget doc is the given widget plus a "variant" property
|
||||
// pulled from a widget.variants string array.
|
||||
return {
|
||||
...widget,
|
||||
variant,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
In alpha versions, a decorator had access to every `IndexableDocument`
|
||||
simultaneously. This is no longer possible in beta versions (precisely to make
|
||||
the indexing process more efficient and performant). You will need to modify
|
||||
your decorator's logic so that it does not need access to every document at
|
||||
once.
|
||||
|
||||
### Rewriting alpha-style search engines for beta
|
||||
|
||||
Search Engines are responsible for both querying and indexing documents to an
|
||||
underlying search engine technology. While the search engine query interface
|
||||
didn't change between alpha and beta versions, the indexing half of the
|
||||
interface _did_ change.
|
||||
|
||||
In alpha versions of the Backstage Search Platform, a search engine implemented
|
||||
an `index` method which took a `type` and an `IndexableDocument` array and was
|
||||
responsible for writing these documents to the underlying search engine.
|
||||
|
||||
In beta versions, the logic encapsulated by the aforementioned `index` method is
|
||||
contained within an object-mode `Writable` stream which expects objects of type
|
||||
`IndexableDocument`. On the search engine class itself, the `index` method is
|
||||
replaced with a `getIndexer` factory method which still takes the `type`, but
|
||||
resolves an instance of the aforementioned `Writable` stream.
|
||||
|
||||
Although you can choose to implement a `Writable` stream from scratch, the
|
||||
`@backstage/plugin-search-backend-node` package provides a
|
||||
`BatchSearchEngineIndexer` class in order to simplify the developer experience.
|
||||
With this base class, which collects documents in batches of a configurable size
|
||||
on your behalf, all that's needed is to transfer your old `index` method logic
|
||||
into the base class' three methods (`initialize`, `index`, and `finalize`), and
|
||||
implement the factory method that instantiates the stream:
|
||||
|
||||
```ts
|
||||
import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';
|
||||
import { SearchEngine } from '@backstage/search-common';
|
||||
export class YourSearchEngineIndexer extends BatchSearchEngineIndexer {
|
||||
constructor({ type }: { type: string }) {
|
||||
// Customize the number of documents passed to the index method per batch.
|
||||
super({ batchSize: 500 });
|
||||
// An imaginary search engine indexing client.
|
||||
this.index = new SomeSearchEngineIndex({ indexName: type });
|
||||
}
|
||||
async initialize() {
|
||||
// Setup logic. Performed once before any documents are consumed.
|
||||
}
|
||||
async index(documents: IndexableDocument[]) {
|
||||
await this.index.batchOf(documents);
|
||||
}
|
||||
async finalize() {
|
||||
// Teardown logic. Performed once after all documents have been consumed.
|
||||
}
|
||||
}
|
||||
export class YourSearchEngine implements SearchEngine {
|
||||
async getIndexer(type: string) {
|
||||
return new YourSearchEngineIndexer({ type });
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
[obj-mode]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_object_mode
|
||||
[read-stream]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_readable_streams
|
||||
[async-gen]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of#iterating_over_async_generators
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
useHotCleanup,
|
||||
} from '@backstage/backend-common';
|
||||
import { Config } from '@backstage/config';
|
||||
import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
|
||||
import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend';
|
||||
import { createRouter } from '@backstage/plugin-search-backend';
|
||||
import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch';
|
||||
import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg';
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
LunrSearchEngine,
|
||||
SearchEngine,
|
||||
} from '@backstage/plugin-search-backend-node';
|
||||
import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
|
||||
import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
|
||||
import { Logger } from 'winston';
|
||||
import { PluginEnvironment } from '../types';
|
||||
|
||||
@@ -70,7 +70,7 @@ export default async function createPlugin({
|
||||
// particular collator gathers entities from the software catalog.
|
||||
indexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 600,
|
||||
collator: DefaultCatalogCollator.fromConfig(config, {
|
||||
factory: DefaultCatalogCollatorFactory.fromConfig(config, {
|
||||
discovery,
|
||||
tokenManager,
|
||||
}),
|
||||
@@ -78,7 +78,7 @@ export default async function createPlugin({
|
||||
|
||||
indexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 600,
|
||||
collator: DefaultTechDocsCollator.fromConfig(config, {
|
||||
factory: DefaultTechDocsCollatorFactory.fromConfig(config, {
|
||||
discovery,
|
||||
logger,
|
||||
tokenManager,
|
||||
|
||||
@@ -5,8 +5,8 @@ import {
|
||||
LunrSearchEngine,
|
||||
} from '@backstage/plugin-search-backend-node';
|
||||
import { PluginEnvironment } from '../types';
|
||||
import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
|
||||
import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
|
||||
import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend';
|
||||
import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
|
||||
|
||||
export default async function createPlugin({
|
||||
logger,
|
||||
@@ -23,7 +23,7 @@ export default async function createPlugin({
|
||||
// collator gathers entities from the software catalog.
|
||||
indexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 600,
|
||||
collator: DefaultCatalogCollator.fromConfig(config, {
|
||||
factory: DefaultCatalogCollatorFactory.fromConfig(config, {
|
||||
discovery,
|
||||
tokenManager,
|
||||
}),
|
||||
@@ -32,7 +32,7 @@ export default async function createPlugin({
|
||||
// collator gathers entities from techdocs.
|
||||
indexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 600,
|
||||
collator: DefaultTechDocsCollator.fromConfig(config, {
|
||||
factory: DefaultTechDocsCollatorFactory.fromConfig(config, {
|
||||
discovery,
|
||||
logger,
|
||||
tokenManager,
|
||||
|
||||
@@ -3,38 +3,33 @@
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
/// <reference types="node" />
|
||||
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { Permission } from '@backstage/plugin-permission-common';
|
||||
import { Readable } from 'stream';
|
||||
import { Transform } from 'stream';
|
||||
import { Writable } from 'stream';
|
||||
|
||||
// Warning: (ae-missing-release-tag) "DocumentCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public
|
||||
export interface DocumentCollator {
|
||||
// (undocumented)
|
||||
execute(): Promise<IndexableDocument[]>;
|
||||
// @beta
|
||||
export interface DocumentCollatorFactory {
|
||||
getCollator(): Promise<Readable>;
|
||||
readonly type: string;
|
||||
readonly visibilityPermission?: Permission;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "DocumentDecorator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public
|
||||
export interface DocumentDecorator {
|
||||
// (undocumented)
|
||||
execute(documents: IndexableDocument[]): Promise<IndexableDocument[]>;
|
||||
// @beta
|
||||
export interface DocumentDecoratorFactory {
|
||||
getDecorator(): Promise<Transform>;
|
||||
readonly types?: string[];
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "DocumentTypeInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public
|
||||
// @beta
|
||||
export type DocumentTypeInfo = {
|
||||
visibilityPermission?: Permission;
|
||||
};
|
||||
|
||||
// Warning: (ae-missing-release-tag) "IndexableDocument" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public
|
||||
// @beta
|
||||
export interface IndexableDocument {
|
||||
authorization?: {
|
||||
resourceRef: string;
|
||||
@@ -44,23 +39,17 @@ export interface IndexableDocument {
|
||||
title: string;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "QueryRequestOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
// @beta
|
||||
export type QueryRequestOptions = {
|
||||
token?: string;
|
||||
};
|
||||
|
||||
// Warning: (ae-missing-release-tag) "QueryTranslator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public
|
||||
// @beta
|
||||
export type QueryTranslator = (query: SearchQuery) => unknown;
|
||||
|
||||
// Warning: (ae-missing-release-tag) "SearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public
|
||||
// @beta
|
||||
export interface SearchEngine {
|
||||
index(type: string, documents: IndexableDocument[]): Promise<void>;
|
||||
getIndexer(type: string): Promise<Writable>;
|
||||
query(
|
||||
query: SearchQuery,
|
||||
options?: QueryRequestOptions,
|
||||
@@ -68,9 +57,7 @@ export interface SearchEngine {
|
||||
setTranslator(translator: QueryTranslator): void;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "SearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
// @beta (undocumented)
|
||||
export interface SearchQuery {
|
||||
// (undocumented)
|
||||
filters?: JsonObject;
|
||||
@@ -82,9 +69,7 @@ export interface SearchQuery {
|
||||
types?: string[];
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "SearchResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
// @beta (undocumented)
|
||||
export interface SearchResult {
|
||||
// (undocumented)
|
||||
document: IndexableDocument;
|
||||
@@ -92,9 +77,7 @@ export interface SearchResult {
|
||||
type: string;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "SearchResultSet" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
// @beta (undocumented)
|
||||
export interface SearchResultSet {
|
||||
// (undocumented)
|
||||
nextPageCursor?: string;
|
||||
|
||||
@@ -16,7 +16,11 @@
|
||||
|
||||
import { Permission } from '@backstage/plugin-permission-common';
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { Readable, Transform, Writable } from 'stream';
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export interface SearchQuery {
|
||||
term: string;
|
||||
filters?: JsonObject;
|
||||
@@ -24,11 +28,17 @@ export interface SearchQuery {
|
||||
pageCursor?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export interface SearchResult {
|
||||
type: string;
|
||||
document: IndexableDocument;
|
||||
}
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export interface SearchResultSet {
|
||||
results: SearchResult[];
|
||||
nextPageCursor?: string;
|
||||
@@ -38,6 +48,7 @@ export interface SearchResultSet {
|
||||
/**
|
||||
* Base properties that all indexed documents must include, as well as some
|
||||
* common properties that documents are encouraged to use where appropriate.
|
||||
* @beta
|
||||
*/
|
||||
export interface IndexableDocument {
|
||||
/**
|
||||
@@ -72,6 +83,7 @@ export interface IndexableDocument {
|
||||
* Information about a specific document type. Intended to be used in the
|
||||
* {@link @backstage/search-backend-node#IndexBuilder} to collect information
|
||||
* about the types stored in the index.
|
||||
* @beta
|
||||
*/
|
||||
export type DocumentTypeInfo = {
|
||||
/**
|
||||
@@ -82,10 +94,10 @@ export type DocumentTypeInfo = {
|
||||
};
|
||||
|
||||
/**
|
||||
* Interface that must be implemented in order to expose new documents to
|
||||
* search.
|
||||
* Factory class for instantiating collators.
|
||||
* @beta
|
||||
*/
|
||||
export interface DocumentCollator {
|
||||
export interface DocumentCollatorFactory {
|
||||
/**
|
||||
* The type or name of the document set returned by this collator. Used as an
|
||||
* index name by Search Engines.
|
||||
@@ -98,29 +110,41 @@ export interface DocumentCollator {
|
||||
*/
|
||||
readonly visibilityPermission?: Permission;
|
||||
|
||||
execute(): Promise<IndexableDocument[]>;
|
||||
/**
|
||||
* Instantiates and resolves a document collator.
|
||||
*/
|
||||
getCollator(): Promise<Readable>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface that must be implemented in order to decorate existing documents with
|
||||
* additional metadata.
|
||||
* Factory class for instantiating decorators.
|
||||
* @beta
|
||||
*/
|
||||
export interface DocumentDecorator {
|
||||
export interface DocumentDecoratorFactory {
|
||||
/**
|
||||
* An optional array of document/index types on which this decorator should
|
||||
* be applied. If no types are provided, this decorator will be applied to
|
||||
* all document/index types.
|
||||
*/
|
||||
readonly types?: string[];
|
||||
execute(documents: IndexableDocument[]): Promise<IndexableDocument[]>;
|
||||
|
||||
/**
|
||||
* Instantiates and resolves a document decorator.
|
||||
*/
|
||||
getDecorator(): Promise<Transform>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A type of function responsible for translating an abstract search query into
|
||||
* a concrete query relevant to a particular search engine.
|
||||
* @beta
|
||||
*/
|
||||
export type QueryTranslator = (query: SearchQuery) => unknown;
|
||||
|
||||
/**
|
||||
* Options when querying a search engine.
|
||||
* @beta
|
||||
*/
|
||||
export type QueryRequestOptions = {
|
||||
token?: string;
|
||||
};
|
||||
@@ -129,6 +153,7 @@ export type QueryRequestOptions = {
|
||||
* Interface that must be implemented by specific search engines, responsible
|
||||
* for performing indexing and querying and translating abstract queries into
|
||||
* concrete, search engine-specific queries.
|
||||
* @beta
|
||||
*/
|
||||
export interface SearchEngine {
|
||||
/**
|
||||
@@ -137,9 +162,15 @@ export interface SearchEngine {
|
||||
setTranslator(translator: QueryTranslator): void;
|
||||
|
||||
/**
|
||||
* Add the given documents to the SearchEngine index of the given type.
|
||||
* Factory method for getting a search engine indexer for a given document
|
||||
* type.
|
||||
*
|
||||
* @param type - The type or name of the document set for which an indexer
|
||||
* should be retrieved. This corresponds to the `type` property on the
|
||||
* document collator/decorator factories and will most often be used to
|
||||
* identify an index or group to which documents should be written.
|
||||
*/
|
||||
index(type: string, documents: IndexableDocument[]): Promise<void>;
|
||||
getIndexer(type: string): Promise<Writable>;
|
||||
|
||||
/**
|
||||
* Perform a search query against the SearchEngine.
|
||||
|
||||
@@ -10,7 +10,7 @@ import { CatalogApi } from '@backstage/catalog-client';
|
||||
import { ConditionalPolicyDecision } from '@backstage/plugin-permission-node';
|
||||
import { Conditions } from '@backstage/plugin-permission-node';
|
||||
import { Config } from '@backstage/config';
|
||||
import { DocumentCollator } from '@backstage/search-common';
|
||||
import { DocumentCollatorFactory } from '@backstage/search-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { EntityName } from '@backstage/catalog-model';
|
||||
import { EntityPolicy } from '@backstage/catalog-model';
|
||||
@@ -30,6 +30,7 @@ import { PermissionCriteria } from '@backstage/plugin-permission-common';
|
||||
import { PermissionRule } from '@backstage/plugin-permission-node';
|
||||
import { PluginDatabaseManager } from '@backstage/backend-common';
|
||||
import { PluginEndpointDiscovery } from '@backstage/backend-common';
|
||||
import { Readable } from 'stream';
|
||||
import { Router } from 'express';
|
||||
import { ScmIntegrationRegistry } from '@backstage/integration';
|
||||
import { TokenManager } from '@backstage/backend-common';
|
||||
@@ -415,8 +416,8 @@ export function createRandomRefreshInterval(options: {
|
||||
// @public
|
||||
export function createRouter(options: RouterOptions): Promise<express.Router>;
|
||||
|
||||
// @public (undocumented)
|
||||
export class DefaultCatalogCollator implements DocumentCollator {
|
||||
// @public @deprecated (undocumented)
|
||||
export class DefaultCatalogCollator {
|
||||
constructor(options: {
|
||||
discovery: PluginEndpointDiscovery;
|
||||
tokenManager: TokenManager;
|
||||
@@ -456,6 +457,31 @@ export class DefaultCatalogCollator implements DocumentCollator {
|
||||
readonly visibilityPermission: Permission;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
|
||||
// (undocumented)
|
||||
static fromConfig(
|
||||
_config: Config,
|
||||
options: DefaultCatalogCollatorFactoryOptions,
|
||||
): DefaultCatalogCollatorFactory;
|
||||
// (undocumented)
|
||||
getCollator(): Promise<Readable>;
|
||||
// (undocumented)
|
||||
readonly type: string;
|
||||
// (undocumented)
|
||||
readonly visibilityPermission: Permission;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type DefaultCatalogCollatorFactoryOptions = {
|
||||
discovery: PluginEndpointDiscovery;
|
||||
tokenManager: TokenManager;
|
||||
locationTemplate?: string;
|
||||
filter?: GetEntitiesRequest['filter'];
|
||||
batchSize?: number;
|
||||
catalogClient?: CatalogApi;
|
||||
};
|
||||
|
||||
// @public (undocumented)
|
||||
export class DefaultCatalogProcessingOrchestrator
|
||||
implements CatalogProcessingOrchestrator
|
||||
|
||||
@@ -73,6 +73,7 @@
|
||||
"@backstage/backend-test-utils": "^0.1.19",
|
||||
"@backstage/cli": "^0.14.1",
|
||||
"@backstage/plugin-permission-common": "^0.5.1",
|
||||
"@backstage/plugin-search-backend-node": "0.4.7",
|
||||
"@backstage/test-utils": "^0.2.6",
|
||||
"@types/core-js": "^2.5.4",
|
||||
"@types/git-url-parse": "^9.0.0",
|
||||
|
||||
@@ -23,7 +23,6 @@ import {
|
||||
stringifyEntityRef,
|
||||
UserEntity,
|
||||
} from '@backstage/catalog-model';
|
||||
import { IndexableDocument, DocumentCollator } from '@backstage/search-common';
|
||||
import { Config } from '@backstage/config';
|
||||
import {
|
||||
CatalogApi,
|
||||
@@ -31,18 +30,14 @@ import {
|
||||
GetEntitiesRequest,
|
||||
} from '@backstage/catalog-client';
|
||||
import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common';
|
||||
import { CatalogEntityDocument } from './DefaultCatalogCollatorFactory';
|
||||
|
||||
/** @public */
|
||||
export interface CatalogEntityDocument extends IndexableDocument {
|
||||
componentType: string;
|
||||
namespace: string;
|
||||
kind: string;
|
||||
lifecycle: string;
|
||||
owner: string;
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export class DefaultCatalogCollator implements DocumentCollator {
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Upgrade to a more recent `@backstage/search-backend-node` and
|
||||
* use `DefaultCatalogCollatorFactory` instead.
|
||||
*/
|
||||
export class DefaultCatalogCollator {
|
||||
protected discovery: PluginEndpointDiscovery;
|
||||
protected locationTemplate: string;
|
||||
protected filter?: GetEntitiesRequest['filter'];
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import {
|
||||
PluginEndpointDiscovery,
|
||||
TokenManager,
|
||||
} from '@backstage/backend-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { TestPipeline } from '@backstage/plugin-search-backend-node';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { Readable } from 'stream';
|
||||
import { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory';
|
||||
|
||||
const server = setupServer();
|
||||
|
||||
const expectedEntities: Entity[] = [
|
||||
{
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: {
|
||||
name: 'test-entity',
|
||||
description: 'The expected description',
|
||||
},
|
||||
spec: {
|
||||
type: 'some-type',
|
||||
lifecycle: 'experimental',
|
||||
owner: 'someone',
|
||||
},
|
||||
},
|
||||
{
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: {
|
||||
title: 'Test Entity',
|
||||
name: 'test-entity-2',
|
||||
description: 'The expected description 2',
|
||||
},
|
||||
spec: {
|
||||
type: 'some-type',
|
||||
lifecycle: 'experimental',
|
||||
owner: 'someone',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('DefaultCatalogCollatorFactory', () => {
|
||||
const config = new ConfigReader({});
|
||||
const mockDiscoveryApi: jest.Mocked<PluginEndpointDiscovery> = {
|
||||
getBaseUrl: jest.fn().mockResolvedValue('http://localhost:7007'),
|
||||
getExternalBaseUrl: jest.fn(),
|
||||
};
|
||||
const mockTokenManager: jest.Mocked<TokenManager> = {
|
||||
getToken: jest.fn().mockResolvedValue({ token: '' }),
|
||||
authenticate: jest.fn(),
|
||||
};
|
||||
const options = {
|
||||
discovery: mockDiscoveryApi,
|
||||
tokenManager: mockTokenManager,
|
||||
};
|
||||
|
||||
beforeAll(() => {
|
||||
server.listen();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
server.use(
|
||||
rest.get('http://localhost:7007/entities', (req, res, ctx) => {
|
||||
if (req.url.searchParams.has('filter')) {
|
||||
const filter = req.url.searchParams.get('filter');
|
||||
if (filter === 'kind=Foo,kind=Bar') {
|
||||
// When filtering on the 'Foo,Bar' kinds we simply return no items, to simulate a filter
|
||||
return res(ctx.json([]));
|
||||
}
|
||||
throw new Error('Unexpected filter parameter');
|
||||
}
|
||||
|
||||
// Imitate offset/limit pagination.
|
||||
const offset = parseInt(req.url.searchParams.get('offset') || '0', 10);
|
||||
const limit = parseInt(req.url.searchParams.get('limit') || '500', 10);
|
||||
return res(ctx.json(expectedEntities.slice(offset, limit + offset)));
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
server.close();
|
||||
});
|
||||
|
||||
afterEach(() => server.resetHandlers());
|
||||
|
||||
it('has expected type', () => {
|
||||
const factory = DefaultCatalogCollatorFactory.fromConfig(config, options);
|
||||
expect(factory.type).toBe('software-catalog');
|
||||
});
|
||||
|
||||
describe('getCollator', () => {
|
||||
let factory: DefaultCatalogCollatorFactory;
|
||||
let collator: Readable;
|
||||
|
||||
beforeEach(async () => {
|
||||
factory = DefaultCatalogCollatorFactory.fromConfig(config, options);
|
||||
collator = await factory.getCollator();
|
||||
});
|
||||
|
||||
it('returns a readable stream', async () => {
|
||||
expect(collator).toBeInstanceOf(Readable);
|
||||
});
|
||||
|
||||
it('fetches from the configured catalog service', async () => {
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog');
|
||||
expect(documents).toHaveLength(expectedEntities.length);
|
||||
});
|
||||
|
||||
it('maps a returned entity to an expected CatalogEntityDocument', async () => {
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
|
||||
expect(documents[0]).toMatchObject({
|
||||
title: expectedEntities[0].metadata.name,
|
||||
location: '/catalog/default/component/test-entity',
|
||||
text: expectedEntities[0].metadata.description,
|
||||
namespace: 'default',
|
||||
componentType: expectedEntities[0]!.spec!.type,
|
||||
lifecycle: expectedEntities[0]!.spec!.lifecycle,
|
||||
owner: expectedEntities[0]!.spec!.owner,
|
||||
authorization: {
|
||||
resourceRef: 'component:default/test-entity',
|
||||
},
|
||||
});
|
||||
expect(documents[1]).toMatchObject({
|
||||
title: expectedEntities[1].metadata.title,
|
||||
location: '/catalog/default/component/test-entity-2',
|
||||
text: expectedEntities[1].metadata.description,
|
||||
namespace: 'default',
|
||||
componentType: expectedEntities[1]!.spec!.type,
|
||||
lifecycle: expectedEntities[1]!.spec!.lifecycle,
|
||||
owner: expectedEntities[1]!.spec!.owner,
|
||||
authorization: {
|
||||
resourceRef: 'component:default/test-entity-2',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('maps a returned entity with a custom locationTemplate', async () => {
|
||||
// Provide an alternate location template.
|
||||
factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), {
|
||||
discovery: mockDiscoveryApi,
|
||||
tokenManager: mockTokenManager,
|
||||
locationTemplate: '/software/:name',
|
||||
});
|
||||
collator = await factory.getCollator();
|
||||
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
expect(documents[0]).toMatchObject({
|
||||
location: '/software/test-entity',
|
||||
});
|
||||
});
|
||||
|
||||
it('allows filtering of the retrieved catalog entities', async () => {
|
||||
// Provide a custom filter.
|
||||
factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), {
|
||||
discovery: mockDiscoveryApi,
|
||||
tokenManager: mockTokenManager,
|
||||
filter: {
|
||||
kind: ['Foo', 'Bar'],
|
||||
},
|
||||
});
|
||||
collator = await factory.getCollator();
|
||||
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
|
||||
// The simulated 'Foo,Bar' filter should return in an empty list
|
||||
expect(documents).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('paginates through catalog entities using batchSize', async () => {
|
||||
factory = DefaultCatalogCollatorFactory.fromConfig(config, {
|
||||
...options,
|
||||
batchSize: 1,
|
||||
});
|
||||
collator = await factory.getCollator();
|
||||
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
|
||||
expect(documents).toHaveLength(expectedEntities.length);
|
||||
expect(documents[0].location).toBe(
|
||||
'/catalog/default/component/test-entity',
|
||||
);
|
||||
expect(documents[1].location).toBe(
|
||||
'/catalog/default/component/test-entity-2',
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,173 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
PluginEndpointDiscovery,
|
||||
TokenManager,
|
||||
} from '@backstage/backend-common';
|
||||
import {
|
||||
CatalogApi,
|
||||
CatalogClient,
|
||||
GetEntitiesRequest,
|
||||
} from '@backstage/catalog-client';
|
||||
import {
|
||||
Entity,
|
||||
stringifyEntityRef,
|
||||
UserEntity,
|
||||
} from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import {
|
||||
DocumentCollatorFactory,
|
||||
IndexableDocument,
|
||||
} from '@backstage/search-common';
|
||||
import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
/** @public */
|
||||
export interface CatalogEntityDocument extends IndexableDocument {
|
||||
componentType: string;
|
||||
namespace: string;
|
||||
kind: string;
|
||||
lifecycle: string;
|
||||
owner: string;
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export type DefaultCatalogCollatorFactoryOptions = {
|
||||
discovery: PluginEndpointDiscovery;
|
||||
tokenManager: TokenManager;
|
||||
locationTemplate?: string;
|
||||
filter?: GetEntitiesRequest['filter'];
|
||||
batchSize?: number;
|
||||
catalogClient?: CatalogApi;
|
||||
};
|
||||
|
||||
/** @public */
|
||||
export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
|
||||
public readonly type: string = 'software-catalog';
|
||||
public readonly visibilityPermission = catalogEntityReadPermission;
|
||||
|
||||
private locationTemplate: string;
|
||||
private filter?: GetEntitiesRequest['filter'];
|
||||
private batchSize: number;
|
||||
private readonly catalogClient: CatalogApi;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
static fromConfig(
|
||||
_config: Config,
|
||||
options: DefaultCatalogCollatorFactoryOptions,
|
||||
) {
|
||||
return new DefaultCatalogCollatorFactory(options);
|
||||
}
|
||||
|
||||
private constructor(options: DefaultCatalogCollatorFactoryOptions) {
|
||||
const {
|
||||
batchSize,
|
||||
discovery,
|
||||
locationTemplate,
|
||||
filter,
|
||||
catalogClient,
|
||||
tokenManager,
|
||||
} = options;
|
||||
|
||||
this.locationTemplate =
|
||||
locationTemplate || '/catalog/:namespace/:kind/:name';
|
||||
this.filter = filter;
|
||||
this.batchSize = batchSize || 500;
|
||||
this.catalogClient =
|
||||
catalogClient || new CatalogClient({ discoveryApi: discovery });
|
||||
this.tokenManager = tokenManager;
|
||||
}
|
||||
|
||||
async getCollator(): Promise<Readable> {
|
||||
return Readable.from(this.execute());
|
||||
}
|
||||
|
||||
private applyArgsToFormat(
|
||||
format: string,
|
||||
args: Record<string, string>,
|
||||
): string {
|
||||
let formatted = format;
|
||||
for (const [key, value] of Object.entries(args)) {
|
||||
formatted = formatted.replace(`:${key}`, value);
|
||||
}
|
||||
return formatted.toLowerCase();
|
||||
}
|
||||
|
||||
private isUserEntity(entity: Entity): entity is UserEntity {
|
||||
return entity.kind.toLocaleUpperCase('en-US') === 'USER';
|
||||
}
|
||||
|
||||
private getDocumentText(entity: Entity): string {
|
||||
let documentText = entity.metadata.description || '';
|
||||
if (this.isUserEntity(entity)) {
|
||||
if (entity.spec?.profile?.displayName && documentText) {
|
||||
// combine displayName and description
|
||||
const displayName = entity.spec?.profile?.displayName;
|
||||
documentText = displayName.concat(' : ', documentText);
|
||||
} else {
|
||||
documentText = entity.spec?.profile?.displayName || documentText;
|
||||
}
|
||||
}
|
||||
return documentText;
|
||||
}
|
||||
|
||||
private async *execute(): AsyncGenerator<CatalogEntityDocument> {
|
||||
const { token } = await this.tokenManager.getToken();
|
||||
let entitiesRetrieved = 0;
|
||||
let moreEntitiesToGet = true;
|
||||
|
||||
// Offset/limit pagination is used on the Catalog Client in order to
|
||||
// limit (and allow some control over) memory used by the search backend
|
||||
// at index-time.
|
||||
while (moreEntitiesToGet) {
|
||||
const entities = (
|
||||
await this.catalogClient.getEntities(
|
||||
{
|
||||
filter: this.filter,
|
||||
limit: this.batchSize,
|
||||
offset: entitiesRetrieved,
|
||||
},
|
||||
{ token },
|
||||
)
|
||||
).items;
|
||||
|
||||
// Control looping through entity batches.
|
||||
moreEntitiesToGet = entities.length === this.batchSize;
|
||||
entitiesRetrieved += entities.length;
|
||||
|
||||
for (const entity of entities) {
|
||||
yield {
|
||||
title: entity.metadata.title ?? entity.metadata.name,
|
||||
location: this.applyArgsToFormat(this.locationTemplate, {
|
||||
namespace: entity.metadata.namespace || 'default',
|
||||
kind: entity.kind,
|
||||
name: entity.metadata.name,
|
||||
}),
|
||||
text: this.getDocumentText(entity),
|
||||
componentType: entity.spec?.type?.toString() || 'other',
|
||||
namespace: entity.metadata.namespace || 'default',
|
||||
kind: entity.kind,
|
||||
lifecycle: (entity.spec?.lifecycle as string) || '',
|
||||
owner: (entity.spec?.owner as string) || '',
|
||||
authorization: {
|
||||
resourceRef: stringifyEntityRef(entity),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,5 +14,11 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory';
|
||||
export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory';
|
||||
export type { CatalogEntityDocument } from './DefaultCatalogCollatorFactory';
|
||||
|
||||
/**
|
||||
* todo(backstage/techdocs-core): stop exporting this in a future release.
|
||||
*/
|
||||
export { DefaultCatalogCollator } from './DefaultCatalogCollator';
|
||||
export type { CatalogEntityDocument } from './DefaultCatalogCollator';
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
```ts
|
||||
/// <reference types="node" />
|
||||
|
||||
import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { Config } from '@backstage/config';
|
||||
import type { ConnectionOptions } from 'tls';
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
@@ -114,7 +116,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
|
||||
indexPrefix,
|
||||
}: ElasticSearchOptions): Promise<ElasticSearchSearchEngine>;
|
||||
// (undocumented)
|
||||
index(type: string, documents: IndexableDocument[]): Promise<void>;
|
||||
getIndexer(type: string): Promise<ElasticSearchSearchEngineIndexer>;
|
||||
newClient<T>(create: (options: ElasticSearchClientOptions) => T): T;
|
||||
// (undocumented)
|
||||
query(query: SearchQuery): Promise<SearchResultSet>;
|
||||
@@ -127,4 +129,31 @@ export class ElasticSearchSearchEngine implements SearchEngine {
|
||||
// (undocumented)
|
||||
protected translator(query: SearchQuery): ConcreteElasticSearchQuery;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
|
||||
constructor(options: ElasticSearchSearchEngineIndexerOptions);
|
||||
// (undocumented)
|
||||
finalize(): Promise<void>;
|
||||
// (undocumented)
|
||||
index(documents: IndexableDocument[]): Promise<void>;
|
||||
// (undocumented)
|
||||
readonly indexName: string;
|
||||
// (undocumented)
|
||||
initialize(): Promise<void>;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngineIndexerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
export type ElasticSearchSearchEngineIndexerOptions = {
|
||||
type: string;
|
||||
indexPrefix: string;
|
||||
indexSeparator: string;
|
||||
alias: string;
|
||||
logger: Logger_2;
|
||||
elasticSearchClient: Client;
|
||||
};
|
||||
```
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
"dependencies": {
|
||||
"@backstage/config": "^0.1.15",
|
||||
"@backstage/search-common": "^0.2.4",
|
||||
"@backstage/plugin-search-backend-node": "^0.4.7",
|
||||
"@elastic/elasticsearch": "7.13.0",
|
||||
"@acuris/aws-es-connection": "^2.2.0",
|
||||
"aws-sdk": "^2.948.0",
|
||||
|
||||
+73
-18
@@ -15,7 +15,8 @@
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { Client, errors } from '@elastic/elasticsearch';
|
||||
import Mock from '@elastic/elasticsearch-mock';
|
||||
import {
|
||||
ConcreteElasticSearchQuery,
|
||||
@@ -23,7 +24,7 @@ import {
|
||||
ElasticSearchSearchEngine,
|
||||
encodePageCursor,
|
||||
} from './ElasticSearchSearchEngine';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer';
|
||||
|
||||
class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEngine {
|
||||
getTranslator() {
|
||||
@@ -37,6 +38,16 @@ const options = {
|
||||
Connection: mock.getConnection(),
|
||||
};
|
||||
|
||||
const indexerMock = {
|
||||
on: jest.fn(),
|
||||
indexName: 'expected-index-name',
|
||||
};
|
||||
jest.mock('./ElasticSearchSearchEngineIndexer', () => ({
|
||||
ElasticSearchSearchEngineIndexer: jest
|
||||
.fn()
|
||||
.mockImplementation(() => indexerMock),
|
||||
}));
|
||||
|
||||
describe('ElasticSearchSearchEngine', () => {
|
||||
let testSearchEngine: ElasticSearchSearchEngine;
|
||||
let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests;
|
||||
@@ -542,23 +553,67 @@ describe('ElasticSearchSearchEngine', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('index', () => {
|
||||
it('should index document', async () => {
|
||||
const indexSpy = jest.spyOn(testSearchEngine, 'index');
|
||||
const mockDocuments = [
|
||||
{
|
||||
title: 'testTerm',
|
||||
text: 'testText',
|
||||
location: 'test/location',
|
||||
},
|
||||
];
|
||||
describe('indexer', () => {
|
||||
it('should get indexer', async () => {
|
||||
const indexer = await testSearchEngine.getIndexer('test-index');
|
||||
|
||||
// call index func and ensure the index func was invoked.
|
||||
await testSearchEngine.index('test-index', mockDocuments);
|
||||
expect(indexSpy).toHaveBeenCalled();
|
||||
expect(indexSpy).toHaveBeenCalledWith('test-index', [
|
||||
{ title: 'testTerm', text: 'testText', location: 'test/location' },
|
||||
]);
|
||||
expect(indexer).toStrictEqual(indexerMock);
|
||||
expect(ElasticSearchSearchEngineIndexer).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
alias: 'test-index__search',
|
||||
type: 'test-index',
|
||||
indexPrefix: '',
|
||||
indexSeparator: '-index__',
|
||||
elasticSearchClient: client,
|
||||
}),
|
||||
);
|
||||
expect(indexerMock.on).toHaveBeenCalledWith(
|
||||
'error',
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
|
||||
describe('onError', () => {
|
||||
let errorHandler: Function;
|
||||
const error = new Error('some error');
|
||||
|
||||
beforeEach(async () => {
|
||||
mock.clearAll();
|
||||
await testSearchEngine.getIndexer('test-index');
|
||||
errorHandler = indexerMock.on.mock.calls[0][1];
|
||||
});
|
||||
|
||||
it('should check for and delete expected index', async () => {
|
||||
const existsSpy = jest.fn().mockReturnValue('truthy value');
|
||||
const deleteSpy = jest.fn().mockReturnValue({});
|
||||
mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy);
|
||||
mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy);
|
||||
|
||||
await errorHandler(error);
|
||||
|
||||
// Check and delete HTTP requests were made.
|
||||
expect(existsSpy).toHaveBeenCalled();
|
||||
expect(deleteSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not delete index if none exists', async () => {
|
||||
// Exists call returns 404 on no index.
|
||||
const existsSpy = jest.fn().mockReturnValue(
|
||||
new errors.ResponseError({
|
||||
statusCode: 404,
|
||||
body: { status: 404 },
|
||||
} as unknown as any),
|
||||
);
|
||||
const deleteSpy = jest.fn().mockReturnValue({});
|
||||
mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy);
|
||||
mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy);
|
||||
|
||||
await errorHandler(error);
|
||||
|
||||
// Check request was made, but no delete request was made.
|
||||
expect(existsSpy).toHaveBeenCalled();
|
||||
expect(deleteSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+27
-67
@@ -29,8 +29,8 @@ import { Client } from '@elastic/elasticsearch';
|
||||
import esb from 'elastic-builder';
|
||||
import { isEmpty, isNaN as nan, isNumber } from 'lodash';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
import type { ElasticSearchClientOptions } from './ElasticSearchClientOptions';
|
||||
import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer';
|
||||
|
||||
export type { ElasticSearchClientOptions };
|
||||
|
||||
@@ -58,12 +58,6 @@ type ElasticSearchResult = {
|
||||
_source: IndexableDocument;
|
||||
};
|
||||
|
||||
function duration(startTimestamp: [number, number]): string {
|
||||
const delta = process.hrtime(startTimestamp);
|
||||
const seconds = delta[0] + delta[1] / 1e9;
|
||||
return `${seconds.toFixed(1)}s`;
|
||||
}
|
||||
|
||||
function isBlank(str: string) {
|
||||
return (isEmpty(str) && !isNumber(str)) || nan(str);
|
||||
}
|
||||
@@ -165,67 +159,37 @@ export class ElasticSearchSearchEngine implements SearchEngine {
|
||||
this.translator = translator;
|
||||
}
|
||||
|
||||
async index(type: string, documents: IndexableDocument[]): Promise<void> {
|
||||
this.logger.info(
|
||||
`Started indexing ${documents.length} documents for index ${type}`,
|
||||
);
|
||||
const startTimestamp = process.hrtime();
|
||||
async getIndexer(type: string) {
|
||||
const alias = this.constructSearchAlias(type);
|
||||
const index = this.constructIndexName(type, `${Date.now()}`);
|
||||
try {
|
||||
const aliases = await this.elasticSearchClient.cat.aliases({
|
||||
format: 'json',
|
||||
name: alias,
|
||||
});
|
||||
const removableIndices = aliases.body.map(
|
||||
(r: Record<string, any>) => r.index,
|
||||
);
|
||||
const indexer = new ElasticSearchSearchEngineIndexer({
|
||||
type,
|
||||
indexPrefix: this.indexPrefix,
|
||||
indexSeparator: this.indexSeparator,
|
||||
alias,
|
||||
elasticSearchClient: this.elasticSearchClient,
|
||||
logger: this.logger,
|
||||
});
|
||||
|
||||
await this.elasticSearchClient.indices.create({
|
||||
index,
|
||||
});
|
||||
const result = await this.elasticSearchClient.helpers.bulk({
|
||||
datasource: documents,
|
||||
onDocument() {
|
||||
return {
|
||||
index: { _index: index },
|
||||
};
|
||||
},
|
||||
refreshOnCompletion: index,
|
||||
});
|
||||
|
||||
this.logger.info(
|
||||
`Indexing completed for index ${type} in ${duration(startTimestamp)}`,
|
||||
result,
|
||||
);
|
||||
await this.elasticSearchClient.indices.updateAliases({
|
||||
body: {
|
||||
actions: [
|
||||
{ remove: { index: this.constructIndexName(type, '*'), alias } },
|
||||
{ add: { index, alias } },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
this.logger.info('Removing stale search indices', removableIndices);
|
||||
if (removableIndices.length) {
|
||||
await this.elasticSearchClient.indices.delete({
|
||||
index: removableIndices,
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
// Attempt cleanup upon failure.
|
||||
indexer.on('error', async e => {
|
||||
this.logger.error(`Failed to index documents for type ${type}`, e);
|
||||
const response = await this.elasticSearchClient.indices.exists({
|
||||
index,
|
||||
});
|
||||
const indexCreated = response.body;
|
||||
if (indexCreated) {
|
||||
this.logger.info(`Removing created index ${index}`);
|
||||
await this.elasticSearchClient.indices.delete({
|
||||
index,
|
||||
try {
|
||||
const response = await this.elasticSearchClient.indices.exists({
|
||||
index: indexer.indexName,
|
||||
});
|
||||
const indexCreated = response.body;
|
||||
if (indexCreated) {
|
||||
this.logger.info(`Removing created index ${indexer.indexName}`);
|
||||
await this.elasticSearchClient.indices.delete({
|
||||
index: indexer.indexName,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`Unable to clean up elastic index: ${error}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return indexer;
|
||||
}
|
||||
|
||||
async query(query: SearchQuery): Promise<SearchResultSet> {
|
||||
@@ -268,10 +232,6 @@ export class ElasticSearchSearchEngine implements SearchEngine {
|
||||
|
||||
private readonly indexSeparator = '-index__';
|
||||
|
||||
private constructIndexName(type: string, postFix: string) {
|
||||
return `${this.indexPrefix}${type}${this.indexSeparator}${postFix}`;
|
||||
}
|
||||
|
||||
private getTypeFromIndex(index: string) {
|
||||
return index
|
||||
.substring(this.indexPrefix.length)
|
||||
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { TestPipeline } from '@backstage/plugin-search-backend-node';
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import Mock from '@elastic/elasticsearch-mock';
|
||||
import { range } from 'lodash';
|
||||
import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer';
|
||||
|
||||
const mock = new Mock();
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: mock.getConnection(),
|
||||
});
|
||||
|
||||
describe('ElasticSearchSearchEngineIndexer', () => {
|
||||
let indexer: ElasticSearchSearchEngineIndexer;
|
||||
let bulkSpy: jest.Mock;
|
||||
let catSpy: jest.Mock;
|
||||
let createSpy: jest.Mock;
|
||||
let aliasesSpy: jest.Mock;
|
||||
let deleteSpy: jest.Mock;
|
||||
|
||||
beforeEach(() => {
|
||||
// Instantiate the indexer to be tested.
|
||||
indexer = new ElasticSearchSearchEngineIndexer({
|
||||
type: 'some-type',
|
||||
indexPrefix: '',
|
||||
indexSeparator: '-index__',
|
||||
alias: 'some-type-index__search',
|
||||
logger: getVoidLogger(),
|
||||
elasticSearchClient: client,
|
||||
});
|
||||
|
||||
// Set up all requisite Elastic mocks.
|
||||
mock.clearAll();
|
||||
bulkSpy = jest.fn().mockReturnValue({ took: 9, errors: false, items: [] });
|
||||
mock.add(
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/_bulk',
|
||||
},
|
||||
bulkSpy,
|
||||
);
|
||||
mock.add(
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/:index/_refresh',
|
||||
},
|
||||
jest.fn().mockReturnValue({}),
|
||||
);
|
||||
|
||||
catSpy = jest.fn().mockReturnValue([
|
||||
{
|
||||
alias: 'some-type-index__search',
|
||||
index: 'some-type-index__123tobedeleted',
|
||||
filter: '-',
|
||||
'routing.index': '-',
|
||||
'routing.search': '-',
|
||||
is_write_index: '-',
|
||||
},
|
||||
]);
|
||||
mock.add(
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/_cat/aliases/some-type-index__search',
|
||||
},
|
||||
catSpy,
|
||||
);
|
||||
|
||||
createSpy = jest.fn().mockReturnValue({
|
||||
acknowledged: true,
|
||||
shards_acknowledged: true,
|
||||
index: 'single_index',
|
||||
});
|
||||
mock.add(
|
||||
{
|
||||
method: 'PUT',
|
||||
path: '/:index',
|
||||
},
|
||||
createSpy,
|
||||
);
|
||||
|
||||
aliasesSpy = jest.fn().mockReturnValue({});
|
||||
mock.add(
|
||||
{
|
||||
method: 'POST',
|
||||
path: '*',
|
||||
},
|
||||
aliasesSpy,
|
||||
);
|
||||
|
||||
deleteSpy = jest.fn().mockReturnValue({});
|
||||
mock.add(
|
||||
{
|
||||
method: 'DELETE',
|
||||
path: '/some-type-index__123tobedeleted',
|
||||
},
|
||||
deleteSpy,
|
||||
);
|
||||
});
|
||||
|
||||
it('indexes documents', async () => {
|
||||
const documents = [
|
||||
{
|
||||
title: 'testTerm',
|
||||
text: 'testText',
|
||||
location: 'test/location',
|
||||
},
|
||||
{
|
||||
title: 'Another test',
|
||||
text: 'Some more text',
|
||||
location: 'test/location/2',
|
||||
},
|
||||
];
|
||||
|
||||
await TestPipeline.withSubject(indexer).withDocuments(documents).execute();
|
||||
|
||||
// Older indices should have been queried for.
|
||||
expect(catSpy).toHaveBeenCalled();
|
||||
|
||||
// A new index should have been created.
|
||||
const createdIndex = createSpy.mock.calls[0][0].path.slice(1);
|
||||
expect(createdIndex).toContain('some-type-index__');
|
||||
|
||||
// Bulk helper should have been called with documents.
|
||||
const bulkBody = bulkSpy.mock.calls[0][0].body;
|
||||
expect(bulkBody[0]).toStrictEqual({ index: { _index: createdIndex } });
|
||||
expect(bulkBody[1]).toStrictEqual(documents[0]);
|
||||
expect(bulkBody[2]).toStrictEqual({ index: { _index: createdIndex } });
|
||||
expect(bulkBody[3]).toStrictEqual(documents[1]);
|
||||
|
||||
// Alias should have been rotated.
|
||||
expect(aliasesSpy).toHaveBeenCalled();
|
||||
const aliasActions = aliasesSpy.mock.calls[0][0].body.actions;
|
||||
expect(aliasActions[0]).toStrictEqual({
|
||||
remove: { index: 'some-type-index__*', alias: 'some-type-index__search' },
|
||||
});
|
||||
expect(aliasActions[1]).toStrictEqual({
|
||||
add: { index: createdIndex, alias: 'some-type-index__search' },
|
||||
});
|
||||
|
||||
// Old index should be cleaned up.
|
||||
expect(deleteSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('handles bulk and batching during indexing', async () => {
|
||||
const documents = range(550).map(i => ({
|
||||
title: `Hello World ${i}`,
|
||||
location: `location-${i}`,
|
||||
// Generate large document sizes to trigger ES bulk flushing.
|
||||
text: range(2000).join(', '),
|
||||
}));
|
||||
|
||||
await TestPipeline.withSubject(indexer).withDocuments(documents).execute();
|
||||
|
||||
// Ensure multiple bulk requests were made.
|
||||
expect(bulkSpy).toHaveBeenCalledTimes(2);
|
||||
|
||||
// Ensure the first and last documents were included in the payloads.
|
||||
const docLocations: string[] = [
|
||||
...bulkSpy.mock.calls[0][0].body.map((l: any) => l.location),
|
||||
...bulkSpy.mock.calls[1][0].body.map((l: any) => l.location),
|
||||
];
|
||||
expect(docLocations).toContain('location-0');
|
||||
expect(docLocations).toContain('location-549');
|
||||
});
|
||||
|
||||
it('ignores cleanup when no existing indices exist', async () => {
|
||||
const documents = [
|
||||
{
|
||||
title: 'testTerm',
|
||||
text: 'testText',
|
||||
location: 'test/location',
|
||||
},
|
||||
];
|
||||
|
||||
// Update initial alias cat to return nothing.
|
||||
catSpy = jest.fn().mockReturnValue([]);
|
||||
mock.clear({
|
||||
method: 'GET',
|
||||
path: '/_cat/aliases/some-type-index__search',
|
||||
});
|
||||
mock.add(
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/_cat/aliases/some-type-index__search',
|
||||
},
|
||||
catSpy,
|
||||
);
|
||||
|
||||
await TestPipeline.withSubject(indexer).withDocuments(documents).execute();
|
||||
|
||||
// Final deletion shouldn't be called.
|
||||
expect(deleteSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { Readable } from 'stream';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
export type ElasticSearchSearchEngineIndexerOptions = {
|
||||
type: string;
|
||||
indexPrefix: string;
|
||||
indexSeparator: string;
|
||||
alias: string;
|
||||
logger: Logger;
|
||||
elasticSearchClient: Client;
|
||||
};
|
||||
|
||||
function duration(startTimestamp: [number, number]): string {
|
||||
const delta = process.hrtime(startTimestamp);
|
||||
const seconds = delta[0] + delta[1] / 1e9;
|
||||
return `${seconds.toFixed(1)}s`;
|
||||
}
|
||||
|
||||
export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
|
||||
private received: number = 0;
|
||||
private processed: number = 0;
|
||||
private removableIndices: string[] = [];
|
||||
|
||||
private readonly startTimestamp: [number, number];
|
||||
private readonly type: string;
|
||||
public readonly indexName: string;
|
||||
private readonly indexPrefix: string;
|
||||
private readonly indexSeparator: string;
|
||||
private readonly alias: string;
|
||||
private readonly logger: Logger;
|
||||
private readonly sourceStream: Readable;
|
||||
private readonly elasticSearchClient: Client;
|
||||
private bulkResult: Promise<any>;
|
||||
|
||||
constructor(options: ElasticSearchSearchEngineIndexerOptions) {
|
||||
super({ batchSize: 100 });
|
||||
this.logger = options.logger;
|
||||
this.startTimestamp = process.hrtime();
|
||||
this.type = options.type;
|
||||
this.indexPrefix = options.indexPrefix;
|
||||
this.indexSeparator = options.indexSeparator;
|
||||
this.indexName = this.constructIndexName(`${Date.now()}`);
|
||||
this.alias = options.alias;
|
||||
this.elasticSearchClient = options.elasticSearchClient;
|
||||
|
||||
// The ES client bulk helper supports stream-based indexing, but we have to
|
||||
// supply the stream directly to it at instantiation-time. We can't supply
|
||||
// this class itself, so instead, we create this inline stream instead.
|
||||
this.sourceStream = new Readable({ objectMode: true });
|
||||
this.sourceStream._read = () => {};
|
||||
|
||||
// eslint-disable-next-line consistent-this
|
||||
const that = this;
|
||||
|
||||
// Keep a reference to the ES Bulk helper so that we can know when all
|
||||
// documents have been successfully written to ES.
|
||||
this.bulkResult = this.elasticSearchClient.helpers.bulk({
|
||||
datasource: this.sourceStream,
|
||||
onDocument() {
|
||||
that.processed++;
|
||||
return {
|
||||
index: { _index: that.indexName },
|
||||
};
|
||||
},
|
||||
refreshOnCompletion: that.indexName,
|
||||
});
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
this.logger.info(`Started indexing documents for index ${this.type}`);
|
||||
|
||||
const aliases = await this.elasticSearchClient.cat.aliases({
|
||||
format: 'json',
|
||||
name: this.alias,
|
||||
});
|
||||
|
||||
this.removableIndices = aliases.body.map(
|
||||
(r: Record<string, any>) => r.index,
|
||||
);
|
||||
|
||||
await this.elasticSearchClient.indices.create({
|
||||
index: this.indexName,
|
||||
});
|
||||
}
|
||||
|
||||
async index(documents: IndexableDocument[]): Promise<void> {
|
||||
await this.isReady();
|
||||
documents.forEach(document => {
|
||||
this.received++;
|
||||
this.sourceStream.push(document);
|
||||
});
|
||||
}
|
||||
|
||||
async finalize(): Promise<void> {
|
||||
// Wait for all documents to be processed.
|
||||
await this.isReady();
|
||||
|
||||
// Close off the underlying stream connected to ES, indicating that no more
|
||||
// documents will be written.
|
||||
this.sourceStream.push(null);
|
||||
|
||||
// Wait for the bulk helper to finish processing.
|
||||
const result = await this.bulkResult;
|
||||
|
||||
// Rotate aliases upon completion. Allow errors to bubble up so that we can
|
||||
// clean up the create index.
|
||||
this.logger.info(
|
||||
`Indexing completed for index ${this.type} in ${duration(
|
||||
this.startTimestamp,
|
||||
)}`,
|
||||
result,
|
||||
);
|
||||
await this.elasticSearchClient.indices.updateAliases({
|
||||
body: {
|
||||
actions: [
|
||||
{
|
||||
remove: { index: this.constructIndexName('*'), alias: this.alias },
|
||||
},
|
||||
{ add: { index: this.indexName, alias: this.alias } },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
// If any indices are removable, remove them. Do not bubble up this error,
|
||||
// as doing so would delete the now aliased index. Log instead.
|
||||
if (this.removableIndices.length) {
|
||||
this.logger.info('Removing stale search indices', this.removableIndices);
|
||||
try {
|
||||
await this.elasticSearchClient.indices.delete({
|
||||
index: this.removableIndices,
|
||||
});
|
||||
} catch (e) {
|
||||
this.logger.warn(`Failed to remove stale search indices: ${e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that the number of documents sent over the wire to ES matches the
|
||||
* number of documents this stream has received so far. This helps manage
|
||||
* backpressure in other parts of the indexing pipeline.
|
||||
*/
|
||||
private isReady(): Promise<void> {
|
||||
return new Promise(resolve => {
|
||||
const interval = setInterval(() => {
|
||||
if (this.received === this.processed) {
|
||||
clearInterval(interval);
|
||||
resolve();
|
||||
}
|
||||
}, 50);
|
||||
});
|
||||
}
|
||||
|
||||
private constructIndexName(postFix: string) {
|
||||
return `${this.indexPrefix}${this.type}${this.indexSeparator}${postFix}`;
|
||||
}
|
||||
}
|
||||
@@ -19,3 +19,7 @@ export type {
|
||||
ConcreteElasticSearchQuery,
|
||||
ElasticSearchClientOptions,
|
||||
} from './ElasticSearchSearchEngine';
|
||||
export type {
|
||||
ElasticSearchSearchEngineIndexer,
|
||||
ElasticSearchSearchEngineIndexerOptions,
|
||||
} from './ElasticSearchSearchEngineIndexer';
|
||||
|
||||
@@ -21,4 +21,8 @@
|
||||
*/
|
||||
|
||||
export { ElasticSearchSearchEngine } from './engines';
|
||||
export type { ElasticSearchClientOptions } from './engines';
|
||||
export type {
|
||||
ElasticSearchClientOptions,
|
||||
ElasticSearchSearchEngineIndexer,
|
||||
ElasticSearchSearchEngineIndexerOptions,
|
||||
} from './engines';
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { Knex } from 'knex';
|
||||
import { PluginDatabaseManager } from '@backstage/backend-common';
|
||||
@@ -28,6 +29,8 @@ export class DatabaseDocumentStore implements DatabaseStore {
|
||||
// (undocumented)
|
||||
static create(knex: Knex): Promise<DatabaseDocumentStore>;
|
||||
// (undocumented)
|
||||
getTransaction(): Promise<Knex.Transaction>;
|
||||
// (undocumented)
|
||||
insertDocuments(
|
||||
tx: Knex.Transaction,
|
||||
type: string,
|
||||
@@ -55,6 +58,8 @@ export interface DatabaseStore {
|
||||
// (undocumented)
|
||||
completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
|
||||
// (undocumented)
|
||||
getTransaction(): Promise<Knex.Transaction>;
|
||||
// (undocumented)
|
||||
insertDocuments(
|
||||
tx: Knex.Transaction,
|
||||
type: string,
|
||||
@@ -81,7 +86,7 @@ export class PgSearchEngine implements SearchEngine {
|
||||
database: PluginDatabaseManager;
|
||||
}): Promise<PgSearchEngine>;
|
||||
// (undocumented)
|
||||
index(type: string, documents: IndexableDocument[]): Promise<void>;
|
||||
getIndexer(type: string): Promise<PgSearchEngineIndexer>;
|
||||
// (undocumented)
|
||||
query(query: SearchQuery): Promise<SearchResultSet>;
|
||||
// (undocumented)
|
||||
@@ -94,6 +99,28 @@ export class PgSearchEngine implements SearchEngine {
|
||||
translator(query: SearchQuery): ConcretePgSearchQuery;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "PgSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
export class PgSearchEngineIndexer extends BatchSearchEngineIndexer {
|
||||
constructor(options: PgSearchEngineIndexerOptions);
|
||||
// (undocumented)
|
||||
finalize(): Promise<void>;
|
||||
// (undocumented)
|
||||
index(documents: IndexableDocument[]): Promise<void>;
|
||||
// (undocumented)
|
||||
initialize(): Promise<void>;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "PgSearchEngineIndexerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
export type PgSearchEngineIndexerOptions = {
|
||||
batchSize: number;
|
||||
type: string;
|
||||
databaseStore: DatabaseStore;
|
||||
};
|
||||
|
||||
// Warning: (ae-missing-release-tag) "PgSearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { range } from 'lodash';
|
||||
import { DatabaseStore } from '../database';
|
||||
import {
|
||||
ConcretePgSearchQuery,
|
||||
@@ -21,6 +20,13 @@ import {
|
||||
encodePageCursor,
|
||||
PgSearchEngine,
|
||||
} from './PgSearchEngine';
|
||||
import { PgSearchEngineIndexer } from './PgSearchEngineIndexer';
|
||||
|
||||
jest.mock('./PgSearchEngineIndexer', () => ({
|
||||
PgSearchEngineIndexer: jest
|
||||
.fn()
|
||||
.mockImplementation(async () => 'the-expected-indexer'),
|
||||
}));
|
||||
|
||||
describe('PgSearchEngine', () => {
|
||||
const tx: any = {} as any;
|
||||
@@ -30,6 +36,7 @@ describe('PgSearchEngine', () => {
|
||||
beforeEach(() => {
|
||||
database = {
|
||||
transaction: jest.fn(),
|
||||
getTransaction: jest.fn(),
|
||||
insertDocuments: jest.fn(),
|
||||
query: jest.fn(),
|
||||
completeInsert: jest.fn(),
|
||||
@@ -122,46 +129,21 @@ describe('PgSearchEngine', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('insert', () => {
|
||||
it('should insert documents', async () => {
|
||||
database.transaction.mockImplementation(fn => fn(tx));
|
||||
describe('index', () => {
|
||||
it('should instantiate indexer', async () => {
|
||||
const indexer = await searchEngine.getIndexer('my-type');
|
||||
|
||||
const documents = [
|
||||
{ title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' },
|
||||
{
|
||||
location: 'location-2',
|
||||
text: 'Hello World',
|
||||
title: 'Dolor sit amet',
|
||||
},
|
||||
];
|
||||
|
||||
await searchEngine.index('my-type', documents);
|
||||
|
||||
expect(database.transaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.prepareInsert).toHaveBeenCalledTimes(1);
|
||||
expect(database.insertDocuments).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'my-type',
|
||||
documents,
|
||||
// Indexer instantiated with expected args.
|
||||
expect(PgSearchEngineIndexer).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
batchSize: 100,
|
||||
type: 'my-type',
|
||||
databaseStore: database,
|
||||
}),
|
||||
);
|
||||
expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
|
||||
});
|
||||
|
||||
it('should batch insert documents', async () => {
|
||||
database.transaction.mockImplementation(fn => fn(tx));
|
||||
|
||||
const documents = range(350).map(i => ({
|
||||
title: `Hello World ${i}`,
|
||||
text: 'Lorem Ipsum',
|
||||
location: `location-${i}`,
|
||||
}));
|
||||
|
||||
await searchEngine.index('my-type', documents);
|
||||
|
||||
expect(database.transaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.prepareInsert).toHaveBeenCalledTimes(1);
|
||||
expect(database.insertDocuments).toBeCalledTimes(4);
|
||||
expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
|
||||
// Indexer is as expected.
|
||||
expect(indexer).toBe('the-expected-indexer');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -15,12 +15,8 @@
|
||||
*/
|
||||
import { PluginDatabaseManager } from '@backstage/backend-common';
|
||||
import { SearchEngine } from '@backstage/plugin-search-backend-node';
|
||||
import {
|
||||
IndexableDocument,
|
||||
SearchQuery,
|
||||
SearchResultSet,
|
||||
} from '@backstage/search-common';
|
||||
import { chunk } from 'lodash';
|
||||
import { SearchQuery, SearchResultSet } from '@backstage/search-common';
|
||||
import { PgSearchEngineIndexer } from './PgSearchEngineIndexer';
|
||||
import {
|
||||
DatabaseDocumentStore,
|
||||
DatabaseStore,
|
||||
@@ -77,16 +73,11 @@ export class PgSearchEngine implements SearchEngine {
|
||||
this.translator = translator;
|
||||
}
|
||||
|
||||
async index(type: string, documents: IndexableDocument[]): Promise<void> {
|
||||
await this.databaseStore.transaction(async tx => {
|
||||
await this.databaseStore.prepareInsert(tx);
|
||||
|
||||
const batchSize = 100;
|
||||
for (const documentBatch of chunk(documents, batchSize)) {
|
||||
await this.databaseStore.insertDocuments(tx, type, documentBatch);
|
||||
}
|
||||
|
||||
await this.databaseStore.completeInsert(tx, type);
|
||||
async getIndexer(type: string) {
|
||||
return new PgSearchEngineIndexer({
|
||||
batchSize: 100,
|
||||
type,
|
||||
databaseStore: this.databaseStore,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { TestPipeline } from '@backstage/plugin-search-backend-node';
|
||||
import { range } from 'lodash';
|
||||
import { PgSearchEngineIndexer } from './PgSearchEngineIndexer';
|
||||
import { DatabaseStore } from '../database';
|
||||
|
||||
describe('PgSearchEngineIndexer', () => {
|
||||
const tx = {
|
||||
rollback: jest.fn(),
|
||||
commit: jest.fn(),
|
||||
} as any;
|
||||
let database: jest.Mocked<DatabaseStore>;
|
||||
let indexer: PgSearchEngineIndexer;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
database = {
|
||||
transaction: jest.fn().mockImplementation(fn => fn(tx)),
|
||||
getTransaction: jest.fn().mockReturnValue(tx),
|
||||
insertDocuments: jest.fn(),
|
||||
query: jest.fn(),
|
||||
completeInsert: jest.fn(),
|
||||
prepareInsert: jest.fn(),
|
||||
};
|
||||
indexer = new PgSearchEngineIndexer({
|
||||
batchSize: 100,
|
||||
type: 'my-type',
|
||||
databaseStore: database,
|
||||
});
|
||||
});
|
||||
|
||||
it('should insert documents', async () => {
|
||||
const documents = [
|
||||
{ title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' },
|
||||
{
|
||||
location: 'location-2',
|
||||
text: 'Hello World',
|
||||
title: 'Dolor sit amet',
|
||||
},
|
||||
];
|
||||
|
||||
await TestPipeline.withSubject(indexer).withDocuments(documents).execute();
|
||||
|
||||
expect(database.getTransaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.prepareInsert).toHaveBeenCalledTimes(1);
|
||||
expect(database.insertDocuments).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'my-type',
|
||||
documents,
|
||||
);
|
||||
expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
|
||||
expect(tx.commit).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should batch insert documents', async () => {
|
||||
const documents = range(350).map(i => ({
|
||||
title: `Hello World ${i}`,
|
||||
text: 'Lorem Ipsum',
|
||||
location: `location-${i}`,
|
||||
}));
|
||||
|
||||
await TestPipeline.withSubject(indexer).withDocuments(documents).execute();
|
||||
|
||||
expect(database.getTransaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.prepareInsert).toHaveBeenCalledTimes(1);
|
||||
expect(database.insertDocuments).toBeCalledTimes(4);
|
||||
expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
|
||||
});
|
||||
|
||||
it('should close out stream and bubble up error on prepare', async () => {
|
||||
const expectedError = new Error('Prepare error');
|
||||
const documents = [
|
||||
{
|
||||
title: `Hello World`,
|
||||
text: 'Lorem Ipsum',
|
||||
location: `location`,
|
||||
},
|
||||
];
|
||||
|
||||
database.prepareInsert.mockRejectedValueOnce(expectedError);
|
||||
const result = await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(documents)
|
||||
.execute();
|
||||
|
||||
expect(database.getTransaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.insertDocuments).not.toHaveBeenCalled();
|
||||
expect(database.completeInsert).not.toHaveBeenCalled();
|
||||
expect(result.error).toBe(expectedError);
|
||||
expect(tx.rollback).toHaveBeenCalledWith(expectedError);
|
||||
});
|
||||
|
||||
it('should close tx and bubble up error on insert', async () => {
|
||||
const expectedError = new Error('Index error');
|
||||
const documents = [
|
||||
{
|
||||
title: `Hello World`,
|
||||
text: 'Lorem Ipsum',
|
||||
location: `location`,
|
||||
},
|
||||
];
|
||||
|
||||
database.insertDocuments.mockRejectedValueOnce(expectedError);
|
||||
const result = await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(documents)
|
||||
.execute();
|
||||
|
||||
expect(database.getTransaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.prepareInsert).toHaveBeenCalledTimes(1);
|
||||
expect(database.completeInsert).not.toHaveBeenCalled();
|
||||
expect(result.error).toBe(expectedError);
|
||||
expect(tx.rollback).toHaveBeenCalledWith(expectedError);
|
||||
});
|
||||
|
||||
it('should close tx and bubble up error on completion', async () => {
|
||||
const expectedError = new Error('Completion error');
|
||||
const documents = [
|
||||
{
|
||||
title: `Hello World`,
|
||||
text: 'Lorem Ipsum',
|
||||
location: `location`,
|
||||
},
|
||||
];
|
||||
|
||||
database.completeInsert.mockRejectedValueOnce(expectedError);
|
||||
const result = await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(documents)
|
||||
.execute();
|
||||
|
||||
expect(database.getTransaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.prepareInsert).toHaveBeenCalledTimes(1);
|
||||
expect(database.insertDocuments).toHaveBeenCalledTimes(1);
|
||||
expect(database.completeInsert).toHaveBeenCalledTimes(1);
|
||||
expect(result.error).toBe(expectedError);
|
||||
expect(tx.rollback).toHaveBeenCalledWith(expectedError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { Knex } from 'knex';
|
||||
import { DatabaseStore } from '../database';
|
||||
|
||||
export type PgSearchEngineIndexerOptions = {
|
||||
batchSize: number;
|
||||
type: string;
|
||||
databaseStore: DatabaseStore;
|
||||
};
|
||||
|
||||
export class PgSearchEngineIndexer extends BatchSearchEngineIndexer {
|
||||
private store: DatabaseStore;
|
||||
private type: string;
|
||||
private tx: Knex.Transaction | undefined;
|
||||
|
||||
constructor(options: PgSearchEngineIndexerOptions) {
|
||||
super({ batchSize: options.batchSize });
|
||||
this.store = options.databaseStore;
|
||||
this.type = options.type;
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
this.tx = await this.store.getTransaction();
|
||||
try {
|
||||
await this.store.prepareInsert(this.tx);
|
||||
} catch (e) {
|
||||
// In case of error, rollback the transaction and re-throw the error so
|
||||
// that the stream can be closed and destroyed properly.
|
||||
this.tx.rollback(e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async index(documents: IndexableDocument[]): Promise<void> {
|
||||
try {
|
||||
await this.store.insertDocuments(this.tx!, this.type, documents);
|
||||
} catch (e) {
|
||||
// In case of error, rollback the transaction and re-throw the error so
|
||||
// that the stream can be closed and destroyed properly.
|
||||
this.tx!.rollback(e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async finalize(): Promise<void> {
|
||||
// Attempt to complete and commit the transaction.
|
||||
try {
|
||||
await this.store.completeInsert(this.tx!, this.type);
|
||||
this.tx!.commit();
|
||||
} catch (e) {
|
||||
// Otherwise, rollback the transaction and re-throw the error so that the
|
||||
// stream can be closed and destroyed properly.
|
||||
this.tx!.rollback!(e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,3 +15,7 @@
|
||||
*/
|
||||
export { PgSearchEngine } from './PgSearchEngine';
|
||||
export type { ConcretePgSearchQuery } from './PgSearchEngine';
|
||||
export type {
|
||||
PgSearchEngineIndexer,
|
||||
PgSearchEngineIndexerOptions,
|
||||
} from './PgSearchEngineIndexer';
|
||||
|
||||
@@ -71,6 +71,10 @@ export class DatabaseDocumentStore implements DatabaseStore {
|
||||
return await this.db.transaction(fn);
|
||||
}
|
||||
|
||||
async getTransaction(): Promise<Knex.Transaction> {
|
||||
return this.db.transaction();
|
||||
}
|
||||
|
||||
async prepareInsert(tx: Knex.Transaction): Promise<void> {
|
||||
// We create a temporary table to collect the hashes of the documents that
|
||||
// we expect to be in the documents table at the end. The table is deleted
|
||||
|
||||
@@ -26,6 +26,7 @@ export interface PgSearchQuery {
|
||||
|
||||
export interface DatabaseStore {
|
||||
transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T>;
|
||||
getTransaction(): Promise<Knex.Transaction>;
|
||||
prepareInsert(tx: Knex.Transaction): Promise<void>;
|
||||
insertDocuments(
|
||||
tx: Knex.Transaction,
|
||||
|
||||
@@ -3,30 +3,60 @@
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
import { DocumentCollator } from '@backstage/search-common';
|
||||
import { DocumentDecorator } from '@backstage/search-common';
|
||||
/// <reference types="node" />
|
||||
|
||||
import { DocumentCollatorFactory } from '@backstage/search-common';
|
||||
import { DocumentDecoratorFactory } from '@backstage/search-common';
|
||||
import { DocumentTypeInfo } from '@backstage/search-common';
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { Logger as Logger_2 } from 'winston';
|
||||
import { default as lunr_2 } from 'lunr';
|
||||
import { QueryTranslator } from '@backstage/search-common';
|
||||
import { Readable } from 'stream';
|
||||
import { SearchEngine } from '@backstage/search-common';
|
||||
import { SearchQuery } from '@backstage/search-common';
|
||||
import { SearchResultSet } from '@backstage/search-common';
|
||||
import { Transform } from 'stream';
|
||||
import { Writable } from 'stream';
|
||||
|
||||
// Warning: (ae-missing-release-tag) "IndexBuilder" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
// @beta
|
||||
export abstract class BatchSearchEngineIndexer extends Writable {
|
||||
constructor(options: BatchSearchEngineOptions);
|
||||
abstract finalize(): Promise<void>;
|
||||
abstract index(documents: IndexableDocument[]): Promise<void>;
|
||||
abstract initialize(): Promise<void>;
|
||||
}
|
||||
|
||||
// @beta (undocumented)
|
||||
export type BatchSearchEngineOptions = {
|
||||
batchSize: number;
|
||||
};
|
||||
|
||||
// @beta (undocumented)
|
||||
export type ConcreteLunrQuery = {
|
||||
lunrQueryBuilder: lunr_2.Index.QueryBuilder;
|
||||
documentTypes?: string[];
|
||||
pageSize: number;
|
||||
};
|
||||
|
||||
// @beta
|
||||
export abstract class DecoratorBase extends Transform {
|
||||
constructor();
|
||||
abstract decorate(
|
||||
document: IndexableDocument,
|
||||
): Promise<IndexableDocument | IndexableDocument[] | undefined>;
|
||||
abstract finalize(): Promise<void>;
|
||||
abstract initialize(): Promise<void>;
|
||||
}
|
||||
|
||||
// @beta (undocumented)
|
||||
export class IndexBuilder {
|
||||
// Warning: (ae-forgotten-export) The symbol "IndexBuilderOptions" needs to be exported by the entry point index.d.ts
|
||||
constructor({ logger, searchEngine }: IndexBuilderOptions);
|
||||
// Warning: (ae-forgotten-export) The symbol "RegisterCollatorParameters" needs to be exported by the entry point index.d.ts
|
||||
addCollator({
|
||||
collator,
|
||||
factory,
|
||||
defaultRefreshIntervalSeconds,
|
||||
}: RegisterCollatorParameters): void;
|
||||
// Warning: (ae-forgotten-export) The symbol "RegisterDecoratorParameters" needs to be exported by the entry point index.d.ts
|
||||
addDecorator({ decorator }: RegisterDecoratorParameters): void;
|
||||
addDecorator({ factory }: RegisterDecoratorParameters): void;
|
||||
build(): Promise<{
|
||||
scheduler: Scheduler;
|
||||
}>;
|
||||
@@ -36,32 +66,61 @@ export class IndexBuilder {
|
||||
getSearchEngine(): SearchEngine;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "LunrSearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
// @beta (undocumented)
|
||||
export type IndexBuilderOptions = {
|
||||
searchEngine: SearchEngine;
|
||||
logger: Logger_2;
|
||||
};
|
||||
|
||||
// @beta (undocumented)
|
||||
export type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery;
|
||||
|
||||
// @beta (undocumented)
|
||||
export class LunrSearchEngine implements SearchEngine {
|
||||
constructor({ logger }: { logger: Logger_2 });
|
||||
// (undocumented)
|
||||
protected docStore: Record<string, IndexableDocument>;
|
||||
// (undocumented)
|
||||
index(type: string, documents: IndexableDocument[]): Promise<void>;
|
||||
getIndexer(type: string): Promise<LunrSearchEngineIndexer>;
|
||||
// (undocumented)
|
||||
protected logger: Logger_2;
|
||||
// (undocumented)
|
||||
protected lunrIndices: Record<string, lunr_2.Index>;
|
||||
// (undocumented)
|
||||
query(query: SearchQuery): Promise<SearchResultSet>;
|
||||
// Warning: (ae-forgotten-export) The symbol "LunrQueryTranslator" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// (undocumented)
|
||||
setTranslator(translator: LunrQueryTranslator): void;
|
||||
// (undocumented)
|
||||
protected translator: QueryTranslator;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "Scheduler" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public
|
||||
// @beta (undocumented)
|
||||
export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {
|
||||
constructor();
|
||||
// (undocumented)
|
||||
buildIndex(): lunr_2.Index;
|
||||
// (undocumented)
|
||||
finalize(): Promise<void>;
|
||||
// (undocumented)
|
||||
getDocumentStore(): Record<string, IndexableDocument>;
|
||||
// (undocumented)
|
||||
index(documents: IndexableDocument[]): Promise<void>;
|
||||
// (undocumented)
|
||||
initialize(): Promise<void>;
|
||||
}
|
||||
|
||||
// @beta
|
||||
export interface RegisterCollatorParameters {
|
||||
defaultRefreshIntervalSeconds: number;
|
||||
factory: DocumentCollatorFactory;
|
||||
}
|
||||
|
||||
// @beta
|
||||
export interface RegisterDecoratorParameters {
|
||||
factory: DocumentDecoratorFactory;
|
||||
}
|
||||
|
||||
// @beta (undocumented)
|
||||
export class Scheduler {
|
||||
constructor({ logger }: { logger: Logger_2 });
|
||||
addToSchedule(task: Function, interval: number): void;
|
||||
@@ -70,4 +129,17 @@ export class Scheduler {
|
||||
}
|
||||
|
||||
export { SearchEngine };
|
||||
|
||||
// @beta
|
||||
export class TestPipeline {
|
||||
execute(): Promise<TestPipelineResult>;
|
||||
withDocuments(documents: IndexableDocument[]): TestPipeline;
|
||||
static withSubject(subject: Readable | Transform | Writable): TestPipeline;
|
||||
}
|
||||
|
||||
// @beta
|
||||
export type TestPipelineResult = {
|
||||
error: unknown;
|
||||
documents: IndexableDocument[];
|
||||
};
|
||||
```
|
||||
|
||||
@@ -23,10 +23,12 @@
|
||||
"clean": "backstage-cli package clean"
|
||||
},
|
||||
"dependencies": {
|
||||
"@backstage/errors": "^0.2.2",
|
||||
"@backstage/search-common": "^0.2.4",
|
||||
"winston": "^3.2.1",
|
||||
"@types/lunr": "^2.3.3",
|
||||
"lodash": "^4.17.21",
|
||||
"lunr": "^2.3.9",
|
||||
"@types/lunr": "^2.3.3"
|
||||
"winston": "^3.2.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/backend-common": "^0.11.0",
|
||||
|
||||
@@ -16,35 +16,37 @@
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import {
|
||||
DocumentCollator,
|
||||
DocumentDecorator,
|
||||
IndexableDocument,
|
||||
DocumentCollatorFactory,
|
||||
DocumentDecoratorFactory,
|
||||
} from '@backstage/search-common';
|
||||
import { Readable, Transform } from 'stream';
|
||||
import { IndexBuilder } from './IndexBuilder';
|
||||
import { LunrSearchEngine, SearchEngine } from './index';
|
||||
|
||||
class TestDocumentCollator implements DocumentCollator {
|
||||
class TestDocumentCollatorFactory implements DocumentCollatorFactory {
|
||||
readonly type: string = 'anything';
|
||||
async execute(): Promise<IndexableDocument[]> {
|
||||
return [];
|
||||
async getCollator(): Promise<Readable> {
|
||||
const collator = new Readable({ objectMode: true });
|
||||
collator._read = () => {};
|
||||
return collator;
|
||||
}
|
||||
}
|
||||
|
||||
class TypedDocumentCollator extends TestDocumentCollator {
|
||||
class TypedDocumentCollatorFactory extends TestDocumentCollatorFactory {
|
||||
readonly type = 'an-expected-type';
|
||||
}
|
||||
|
||||
class TestDocumentDecorator implements DocumentDecorator {
|
||||
async execute(documents: IndexableDocument[]) {
|
||||
return documents;
|
||||
class TestDocumentDecoratorFactory implements DocumentDecoratorFactory {
|
||||
async getDecorator(): Promise<Transform> {
|
||||
return new Transform();
|
||||
}
|
||||
}
|
||||
|
||||
class TypedDocumentDecorator extends TestDocumentDecorator {
|
||||
class TypedDocumentDecoratorFactory extends TestDocumentDecoratorFactory {
|
||||
readonly types = ['an-expected-type'];
|
||||
}
|
||||
|
||||
class DifferentlyTypedDocumentDecorator extends TestDocumentDecorator {
|
||||
class DifferentlyTypedDocumentDecoratorFactory extends TestDocumentDecoratorFactory {
|
||||
readonly types = ['not-the-expected-type'];
|
||||
}
|
||||
|
||||
@@ -64,13 +66,13 @@ describe('IndexBuilder', () => {
|
||||
describe('addCollator', () => {
|
||||
it('adds a collator', async () => {
|
||||
jest.useFakeTimers();
|
||||
const testCollator = new TestDocumentCollator();
|
||||
const collatorSpy = jest.spyOn(testCollator, 'execute');
|
||||
const testCollatorFactory = new TestDocumentCollatorFactory();
|
||||
const collatorSpy = jest.spyOn(testCollatorFactory, 'getCollator');
|
||||
|
||||
// Add a collator.
|
||||
testIndexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 6,
|
||||
collator: testCollator,
|
||||
factory: testCollatorFactory,
|
||||
});
|
||||
|
||||
// Build the index and ensure the collator was invoked.
|
||||
@@ -84,19 +86,19 @@ describe('IndexBuilder', () => {
|
||||
describe('addDecorator', () => {
|
||||
it('adds a decorator', async () => {
|
||||
jest.useFakeTimers();
|
||||
const testCollator = new TestDocumentCollator();
|
||||
const testDecorator = new TestDocumentDecorator();
|
||||
const decoratorSpy = jest.spyOn(testDecorator, 'execute');
|
||||
const testCollatorFactory = new TestDocumentCollatorFactory();
|
||||
const testDecoratorFactory = new TestDocumentDecoratorFactory();
|
||||
const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator');
|
||||
|
||||
// Add a collator.
|
||||
testIndexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 6,
|
||||
collator: testCollator,
|
||||
factory: testCollatorFactory,
|
||||
});
|
||||
|
||||
// Add a decorator.
|
||||
testIndexBuilder.addDecorator({
|
||||
decorator: testDecorator,
|
||||
factory: testDecoratorFactory,
|
||||
});
|
||||
|
||||
// Build the index and ensure the decorator was invoked.
|
||||
@@ -110,27 +112,20 @@ describe('IndexBuilder', () => {
|
||||
|
||||
it('adds a type-specific decorator', async () => {
|
||||
jest.useFakeTimers();
|
||||
const testCollator = new TypedDocumentCollator();
|
||||
const testDecorator = new TypedDocumentDecorator();
|
||||
const docFixture = {
|
||||
title: 'Test',
|
||||
text: 'Test text.',
|
||||
location: '/test/location',
|
||||
};
|
||||
jest
|
||||
.spyOn(testCollator, 'execute')
|
||||
.mockImplementation(async () => [docFixture]);
|
||||
const decoratorSpy = jest.spyOn(testDecorator, 'execute');
|
||||
const testCollatorFactory = new TypedDocumentCollatorFactory();
|
||||
const testDecoratorFactory = new TypedDocumentDecoratorFactory();
|
||||
jest.spyOn(testCollatorFactory, 'getCollator');
|
||||
const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator');
|
||||
|
||||
// Add a collator.
|
||||
testIndexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 6,
|
||||
collator: testCollator,
|
||||
factory: testCollatorFactory,
|
||||
});
|
||||
|
||||
// Add a decorator for the same type.
|
||||
testIndexBuilder.addDecorator({
|
||||
decorator: testDecorator,
|
||||
factory: testDecoratorFactory,
|
||||
});
|
||||
|
||||
// Build the index and ensure the decorator was invoked.
|
||||
@@ -140,31 +135,24 @@ describe('IndexBuilder', () => {
|
||||
// wait for async decorator execution
|
||||
await Promise.resolve();
|
||||
expect(decoratorSpy).toHaveBeenCalled();
|
||||
expect(decoratorSpy).toHaveBeenCalledWith([docFixture]);
|
||||
});
|
||||
|
||||
it('adds a type-specific decorator that should not be called', async () => {
|
||||
const docFixture = {
|
||||
title: 'Test',
|
||||
text: 'Test text.',
|
||||
location: '/test/location',
|
||||
};
|
||||
const testCollator = new TestDocumentCollator();
|
||||
const testDecorator = new DifferentlyTypedDocumentDecorator();
|
||||
const collatorSpy = jest
|
||||
.spyOn(testCollator, 'execute')
|
||||
.mockImplementation(async () => [docFixture]);
|
||||
const decoratorSpy = jest.spyOn(testDecorator, 'execute');
|
||||
const testCollatorFactory = new TestDocumentCollatorFactory();
|
||||
const testDecoratorFactory =
|
||||
new DifferentlyTypedDocumentDecoratorFactory();
|
||||
const collatorSpy = jest.spyOn(testCollatorFactory, 'getCollator');
|
||||
const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator');
|
||||
|
||||
// Add a collator.
|
||||
testIndexBuilder.addCollator({
|
||||
defaultRefreshIntervalSeconds: 6,
|
||||
collator: testCollator,
|
||||
factory: testCollatorFactory,
|
||||
});
|
||||
|
||||
// Add a decorator for a different type.
|
||||
testIndexBuilder.addDecorator({
|
||||
decorator: testDecorator,
|
||||
factory: testDecoratorFactory,
|
||||
});
|
||||
|
||||
// Build the index and ensure the decorator was not invoked.
|
||||
|
||||
@@ -15,32 +15,31 @@
|
||||
*/
|
||||
|
||||
import {
|
||||
DocumentCollator,
|
||||
DocumentDecorator,
|
||||
DocumentCollatorFactory,
|
||||
DocumentDecoratorFactory,
|
||||
DocumentTypeInfo,
|
||||
IndexableDocument,
|
||||
SearchEngine,
|
||||
} from '@backstage/search-common';
|
||||
import { Transform, pipeline } from 'stream';
|
||||
import { Logger } from 'winston';
|
||||
import { Scheduler } from './index';
|
||||
import {
|
||||
IndexBuilderOptions,
|
||||
RegisterCollatorParameters,
|
||||
RegisterDecoratorParameters,
|
||||
} from './types';
|
||||
|
||||
interface CollatorEnvelope {
|
||||
collate: DocumentCollator;
|
||||
factory: DocumentCollatorFactory;
|
||||
refreshInterval: number;
|
||||
}
|
||||
|
||||
type IndexBuilderOptions = {
|
||||
searchEngine: SearchEngine;
|
||||
logger: Logger;
|
||||
};
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export class IndexBuilder {
|
||||
private collators: Record<string, CollatorEnvelope>;
|
||||
private decorators: Record<string, DocumentDecorator[]>;
|
||||
private decorators: Record<string, DocumentDecoratorFactory[]>;
|
||||
private documentTypes: Record<string, DocumentTypeInfo>;
|
||||
private searchEngine: SearchEngine;
|
||||
private logger: Logger;
|
||||
@@ -66,18 +65,18 @@ export class IndexBuilder {
|
||||
* given refresh interval.
|
||||
*/
|
||||
addCollator({
|
||||
collator,
|
||||
factory,
|
||||
defaultRefreshIntervalSeconds,
|
||||
}: RegisterCollatorParameters): void {
|
||||
this.logger.info(
|
||||
`Added ${collator.constructor.name} collator for type ${collator.type}`,
|
||||
`Added ${factory.constructor.name} collator factory for type ${factory.type}`,
|
||||
);
|
||||
this.collators[collator.type] = {
|
||||
this.collators[factory.type] = {
|
||||
refreshInterval: defaultRefreshIntervalSeconds,
|
||||
collate: collator,
|
||||
factory,
|
||||
};
|
||||
this.documentTypes[collator.type] = {
|
||||
visibilityPermission: collator.visibilityPermission,
|
||||
this.documentTypes[factory.type] = {
|
||||
visibilityPermission: factory.visibilityPermission,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -86,18 +85,18 @@ export class IndexBuilder {
|
||||
* the decorator, it will be applied to documents from all known collators,
|
||||
* otherwise it will only be applied to documents of the given types.
|
||||
*/
|
||||
addDecorator({ decorator }: RegisterDecoratorParameters): void {
|
||||
const types = decorator.types || ['*'];
|
||||
addDecorator({ factory }: RegisterDecoratorParameters): void {
|
||||
const types = factory.types || ['*'];
|
||||
this.logger.info(
|
||||
`Added decorator ${decorator.constructor.name} to types ${types.join(
|
||||
`Added decorator ${factory.constructor.name} to types ${types.join(
|
||||
', ',
|
||||
)}`,
|
||||
);
|
||||
types.forEach(type => {
|
||||
if (this.decorators.hasOwnProperty(type)) {
|
||||
this.decorators[type].push(decorator);
|
||||
this.decorators[type].push(factory);
|
||||
} else {
|
||||
this.decorators[type] = [decorator];
|
||||
this.decorators[type] = [factory];
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -111,46 +110,43 @@ export class IndexBuilder {
|
||||
|
||||
Object.keys(this.collators).forEach(type => {
|
||||
scheduler.addToSchedule(async () => {
|
||||
// Collate, Decorate, Index.
|
||||
const decorators: DocumentDecorator[] = (
|
||||
this.decorators['*'] || []
|
||||
).concat(this.decorators[type] || []);
|
||||
|
||||
this.logger.debug(
|
||||
`Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`,
|
||||
// Instantiate the collator.
|
||||
const collator = await this.collators[type].factory.getCollator();
|
||||
this.logger.info(
|
||||
`Collating documents for ${type} via ${this.collators[type].factory.constructor.name}`,
|
||||
);
|
||||
let documents: IndexableDocument[];
|
||||
|
||||
try {
|
||||
documents = await this.collators[type].collate.execute();
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`Collating documents for ${type} via ${this.collators[type].collate.constructor.name} failed: ${e}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Instantiate all relevant decorators.
|
||||
const decorators: Transform[] = await Promise.all(
|
||||
(this.decorators['*'] || [])
|
||||
.concat(this.decorators[type] || [])
|
||||
.map(async factory => {
|
||||
const decorator = await factory.getDecorator();
|
||||
this.logger.info(
|
||||
`Attached decorator via ${factory.constructor.name} to ${type} index pipeline.`,
|
||||
);
|
||||
return decorator;
|
||||
}),
|
||||
);
|
||||
|
||||
for (let i = 0; i < decorators.length; i++) {
|
||||
this.logger.debug(
|
||||
`Decorating ${type} documents via ${decorators[i].constructor.name}`,
|
||||
);
|
||||
try {
|
||||
documents = await decorators[i].execute(documents);
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`Decorating ${type} documents via ${decorators[i].constructor.name} failed: ${e}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Instantiate the indexer.
|
||||
const indexer = await this.searchEngine.getIndexer(type);
|
||||
|
||||
if (!documents || documents.length === 0) {
|
||||
this.logger.debug(`No documents for type "${type}" to index`);
|
||||
return;
|
||||
}
|
||||
// Compose collator/decorators/indexer into a pipeline
|
||||
return new Promise<void>(done => {
|
||||
pipeline([collator, ...decorators, indexer], error => {
|
||||
if (error) {
|
||||
this.logger.error(
|
||||
`Collating documents for ${type} failed: ${error}`,
|
||||
);
|
||||
} else {
|
||||
this.logger.info(`Collating documents for ${type} succeeded`);
|
||||
}
|
||||
|
||||
// pushing documents to index to a configured search engine.
|
||||
await this.searchEngine.index(type, documents);
|
||||
// Signal index pipeline completion!
|
||||
done();
|
||||
});
|
||||
});
|
||||
}, this.collators[type].refreshInterval * 1000);
|
||||
});
|
||||
|
||||
|
||||
@@ -26,6 +26,9 @@ type TaskEnvelope = {
|
||||
* TODO: coordination, error handling
|
||||
*/
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export class Scheduler {
|
||||
private logger: Logger;
|
||||
private schedule: TaskEnvelope[];
|
||||
|
||||
@@ -16,28 +16,61 @@
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import lunr from 'lunr';
|
||||
import { SearchEngine } from '@backstage/search-common';
|
||||
import { IndexableDocument, SearchEngine } from '@backstage/search-common';
|
||||
import {
|
||||
ConcreteLunrQuery,
|
||||
LunrSearchEngine,
|
||||
decodePageCursor,
|
||||
encodePageCursor,
|
||||
} from './LunrSearchEngine';
|
||||
import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer';
|
||||
import { TestPipeline } from '../test-utils';
|
||||
|
||||
/**
|
||||
* Just used to test the default translator shipped with LunrSearchEngine.
|
||||
*/
|
||||
class LunrSearchEngineForTranslatorTests extends LunrSearchEngine {
|
||||
class LunrSearchEngineForTests extends LunrSearchEngine {
|
||||
getDocStore() {
|
||||
return this.docStore;
|
||||
}
|
||||
setDocStore(docStore: Record<string, IndexableDocument>) {
|
||||
this.docStore = docStore;
|
||||
}
|
||||
getLunrIndices() {
|
||||
return this.lunrIndices;
|
||||
}
|
||||
getTranslator() {
|
||||
return this.translator;
|
||||
}
|
||||
}
|
||||
|
||||
const indexerMock = {
|
||||
on: jest.fn(),
|
||||
buildIndex: jest.fn(),
|
||||
getDocumentStore: jest.fn(),
|
||||
};
|
||||
jest.mock('./LunrSearchEngineIndexer', () => ({
|
||||
LunrSearchEngineIndexer: jest.fn().mockImplementation(() => indexerMock),
|
||||
}));
|
||||
|
||||
const getActualIndexer = (engine: SearchEngine, index: string) => {
|
||||
(LunrSearchEngineIndexer as unknown as jest.Mock).mockImplementationOnce(
|
||||
() => {
|
||||
const ActualIndexer = jest.requireActual(
|
||||
'./LunrSearchEngineIndexer',
|
||||
).LunrSearchEngineIndexer;
|
||||
return new ActualIndexer();
|
||||
},
|
||||
);
|
||||
return engine.getIndexer(index);
|
||||
};
|
||||
|
||||
describe('LunrSearchEngine', () => {
|
||||
let testLunrSearchEngine: SearchEngine;
|
||||
|
||||
beforeEach(() => {
|
||||
testLunrSearchEngine = new LunrSearchEngine({ logger: getVoidLogger() });
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('translator', () => {
|
||||
@@ -65,7 +98,7 @@ describe('LunrSearchEngine', () => {
|
||||
});
|
||||
|
||||
it('should return translated query', async () => {
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTests({
|
||||
logger: getVoidLogger(),
|
||||
});
|
||||
const translatorUnderTest = inspectableSearchEngine.getTranslator();
|
||||
@@ -107,7 +140,7 @@ describe('LunrSearchEngine', () => {
|
||||
});
|
||||
|
||||
it('should have default offset and limit', async () => {
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTests({
|
||||
logger: getVoidLogger(),
|
||||
});
|
||||
const translatorUnderTest = inspectableSearchEngine.getTranslator();
|
||||
@@ -148,7 +181,7 @@ describe('LunrSearchEngine', () => {
|
||||
});
|
||||
|
||||
it('should return translated query with 1 filter', async () => {
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTests({
|
||||
logger: getVoidLogger(),
|
||||
});
|
||||
const translatorUnderTest = inspectableSearchEngine.getTranslator();
|
||||
@@ -193,7 +226,7 @@ describe('LunrSearchEngine', () => {
|
||||
});
|
||||
|
||||
it('should handle single-item array filter as scalar value', async () => {
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTests({
|
||||
logger: getVoidLogger(),
|
||||
});
|
||||
const translatorUnderTest = inspectableSearchEngine.getTranslator();
|
||||
@@ -224,7 +257,7 @@ describe('LunrSearchEngine', () => {
|
||||
});
|
||||
|
||||
it('should return translated query with multiple filters', async () => {
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTests({
|
||||
logger: getVoidLogger(),
|
||||
});
|
||||
const translatorUnderTest = inspectableSearchEngine.getTranslator();
|
||||
@@ -273,7 +306,7 @@ describe('LunrSearchEngine', () => {
|
||||
});
|
||||
|
||||
it('should throw if translated query references missing field', async () => {
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTests({
|
||||
logger: getVoidLogger(),
|
||||
});
|
||||
const translatorUnderTest = inspectableSearchEngine.getTranslator();
|
||||
@@ -334,7 +367,13 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock indexing of 1 document
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
// Perform search query
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -359,7 +398,13 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock indexing of 1 document
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
// Perform search query
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -392,7 +437,13 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock indexing of 1 document
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
// Perform search query
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -424,7 +475,13 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock indexing of 1 document
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
// Perform search query
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -456,7 +513,13 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock indexing of 1 document
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
// Perform search query
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -489,7 +552,13 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock indexing of 1 document
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
// Perform search query
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -522,7 +591,13 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock indexing of 1 document
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
// Perform search query
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -560,7 +635,13 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock indexing of 2 documents
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
// Perform search query
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -604,8 +685,21 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock 2 indices with 1 document each
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
await testLunrSearchEngine.index('test-index-2', mockDocuments2);
|
||||
const indexer1 = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
const indexer2 = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index-2',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer1)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
await TestPipeline.withSubject(indexer2)
|
||||
.withDocuments(mockDocuments2)
|
||||
.execute();
|
||||
|
||||
// Perform search query scoped to "test-index-2" with a filter on the field "extraField"
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
term: 'testTitle',
|
||||
@@ -642,7 +736,13 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock indexing of 2 documents
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
// Perform search query
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -695,8 +795,20 @@ describe('LunrSearchEngine', () => {
|
||||
];
|
||||
|
||||
// Mock 2 indices with 2 documents each
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
await testLunrSearchEngine.index('test-index-2', mockDocuments2);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
const indexer2 = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index-2',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer2)
|
||||
.withDocuments(mockDocuments2)
|
||||
.execute();
|
||||
|
||||
// Perform search query scoped to "test-index-2"
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
@@ -734,7 +846,13 @@ describe('LunrSearchEngine', () => {
|
||||
location: `test/location/${i}`,
|
||||
}));
|
||||
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(
|
||||
testLunrSearchEngine,
|
||||
'test-index',
|
||||
);
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
term: 'testTitle',
|
||||
@@ -767,7 +885,10 @@ describe('LunrSearchEngine', () => {
|
||||
location: `test/location/${i}`,
|
||||
}));
|
||||
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
const indexer = await getActualIndexer(testLunrSearchEngine, 'test-index');
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments(mockDocuments)
|
||||
.execute();
|
||||
|
||||
const mockedSearchResult = await testLunrSearchEngine.query({
|
||||
term: 'testTitle',
|
||||
@@ -793,22 +914,46 @@ describe('LunrSearchEngine', () => {
|
||||
});
|
||||
|
||||
describe('index', () => {
|
||||
it('should index document', async () => {
|
||||
const indexSpy = jest.spyOn(testLunrSearchEngine, 'index');
|
||||
const mockDocuments = [
|
||||
{
|
||||
title: 'testTerm',
|
||||
text: 'testText',
|
||||
location: 'test/location',
|
||||
},
|
||||
];
|
||||
it('should get indexer', async () => {
|
||||
const indexer = await testLunrSearchEngine.getIndexer('test-index');
|
||||
expect(LunrSearchEngineIndexer).toHaveBeenCalled();
|
||||
expect(indexer.on).toHaveBeenCalledWith('close', expect.any(Function));
|
||||
});
|
||||
|
||||
// call index func and ensure the index func was invoked.
|
||||
await testLunrSearchEngine.index('test-index', mockDocuments);
|
||||
expect(indexSpy).toHaveBeenCalled();
|
||||
expect(indexSpy).toHaveBeenCalledWith('test-index', [
|
||||
{ title: 'testTerm', text: 'testText', location: 'test/location' },
|
||||
]);
|
||||
it('should manage indices and docs on close', async () => {
|
||||
const doc = { title: 'A doc', text: 'test', location: 'some-location' };
|
||||
|
||||
// Set up an inspectable search engine to pre-set some data.
|
||||
const inspectableSearchEngine = new LunrSearchEngineForTests({
|
||||
logger: getVoidLogger(),
|
||||
});
|
||||
inspectableSearchEngine.setDocStore({ 'existing-location': doc });
|
||||
|
||||
// Mock methds called by close handler.
|
||||
indexerMock.buildIndex.mockReturnValueOnce('expected-index');
|
||||
indexerMock.getDocumentStore.mockReturnValueOnce({
|
||||
'new-location': doc,
|
||||
});
|
||||
|
||||
// Get the indexer and invoke its close handler.
|
||||
await inspectableSearchEngine.getIndexer('test-index');
|
||||
const onClose = indexerMock.on.mock.calls[0][1] as Function;
|
||||
onClose();
|
||||
|
||||
// Ensure mocked methods were called.
|
||||
expect(indexerMock.buildIndex).toHaveBeenCalled();
|
||||
expect(indexerMock.getDocumentStore).toHaveBeenCalled();
|
||||
|
||||
// Ensure the lunr index was written to the search engine.
|
||||
expect(inspectableSearchEngine.getLunrIndices()).toStrictEqual({
|
||||
'test-index': 'expected-index',
|
||||
});
|
||||
|
||||
// Ensure documents are merged into the existing store.
|
||||
expect(inspectableSearchEngine.getDocStore()).toStrictEqual({
|
||||
'existing-location': doc,
|
||||
'new-location': doc,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -23,7 +23,11 @@ import {
|
||||
} from '@backstage/search-common';
|
||||
import lunr from 'lunr';
|
||||
import { Logger } from 'winston';
|
||||
import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer';
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export type ConcreteLunrQuery = {
|
||||
lunrQueryBuilder: lunr.Index.QueryBuilder;
|
||||
documentTypes?: string[];
|
||||
@@ -35,8 +39,14 @@ type LunrResultEnvelope = {
|
||||
type: string;
|
||||
};
|
||||
|
||||
type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery;
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery;
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export class LunrSearchEngine implements SearchEngine {
|
||||
protected lunrIndices: Record<string, lunr.Index> = {};
|
||||
protected docStore: Record<string, IndexableDocument>;
|
||||
@@ -124,30 +134,17 @@ export class LunrSearchEngine implements SearchEngine {
|
||||
this.translator = translator;
|
||||
}
|
||||
|
||||
async index(type: string, documents: IndexableDocument[]): Promise<void> {
|
||||
const lunrBuilder = new lunr.Builder();
|
||||
async getIndexer(type: string) {
|
||||
const indexer = new LunrSearchEngineIndexer();
|
||||
|
||||
lunrBuilder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer);
|
||||
lunrBuilder.searchPipeline.add(lunr.stemmer);
|
||||
|
||||
// Make this lunr index aware of all relevant fields.
|
||||
Object.keys(documents[0]).forEach(field => {
|
||||
lunrBuilder.field(field);
|
||||
indexer.on('close', () => {
|
||||
// Once the stream is closed, build the index and store the documents in
|
||||
// memory for later retrieval.
|
||||
this.lunrIndices[type] = indexer.buildIndex();
|
||||
this.docStore = { ...this.docStore, ...indexer.getDocumentStore() };
|
||||
});
|
||||
|
||||
// Set "location" field as reference field
|
||||
lunrBuilder.ref('location');
|
||||
|
||||
documents.forEach((document: IndexableDocument) => {
|
||||
// Add document to Lunar index
|
||||
lunrBuilder.add(document);
|
||||
// Store documents in memory to be able to look up document using the ref during query time
|
||||
// This is not how you should implement your SearchEngine implementation! Do not copy!
|
||||
this.docStore[document.location] = document;
|
||||
});
|
||||
|
||||
// "Rotate" the index by simply overwriting any existing index of the same name.
|
||||
this.lunrIndices[type] = lunrBuilder.build();
|
||||
return indexer;
|
||||
}
|
||||
|
||||
async query(query: SearchQuery): Promise<SearchResultSet> {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import lunr from 'lunr';
|
||||
import { range } from 'lodash';
|
||||
import { TestPipeline } from '../test-utils';
|
||||
import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer';
|
||||
|
||||
const lunrBuilderAddSpy = jest.fn();
|
||||
const lunrBuilderRefSpy = jest.fn();
|
||||
const lunrBuilderFieldSpy = jest.fn();
|
||||
const lunrBuilderPipelineAddSpy = jest.fn();
|
||||
const lunrBuilderSearchPipelineAddSpy = jest.fn();
|
||||
|
||||
jest.mock('lunr', () => {
|
||||
const actualLunr = jest.requireActual('lunr');
|
||||
return {
|
||||
...actualLunr,
|
||||
Builder: jest.fn().mockImplementation(() => {
|
||||
const actualBuilder = new actualLunr.Builder();
|
||||
actualBuilder.add = lunrBuilderAddSpy;
|
||||
actualBuilder.ref = lunrBuilderRefSpy;
|
||||
actualBuilder.field = lunrBuilderFieldSpy;
|
||||
actualBuilder.pipeline.add = lunrBuilderPipelineAddSpy;
|
||||
actualBuilder.searchPipeline.add = lunrBuilderSearchPipelineAddSpy;
|
||||
return actualBuilder;
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
describe('LunrSearchEngineIndexer', () => {
|
||||
let indexer: LunrSearchEngineIndexer;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
indexer = new LunrSearchEngineIndexer();
|
||||
});
|
||||
|
||||
it('should index documents', async () => {
|
||||
const documents = [
|
||||
{
|
||||
title: 'testTerm',
|
||||
text: 'testText',
|
||||
location: 'test/location',
|
||||
},
|
||||
];
|
||||
|
||||
await TestPipeline.withSubject(indexer).withDocuments(documents).execute();
|
||||
|
||||
expect(lunrBuilderAddSpy).toHaveBeenCalledWith(documents[0]);
|
||||
});
|
||||
|
||||
it('should index documents in bulk', async () => {
|
||||
const documents = range(350).map(i => ({
|
||||
title: `Hello World ${i}`,
|
||||
text: 'Lorem Ipsum',
|
||||
location: `location-${i}`,
|
||||
}));
|
||||
|
||||
await TestPipeline.withSubject(indexer).withDocuments(documents).execute();
|
||||
expect(lunrBuilderAddSpy).toHaveBeenCalledTimes(350);
|
||||
});
|
||||
|
||||
it('should initialize schema', async () => {
|
||||
const documents = [
|
||||
{
|
||||
title: 'testTerm',
|
||||
text: 'testText',
|
||||
location: 'test/location',
|
||||
extra: 'field',
|
||||
},
|
||||
];
|
||||
|
||||
await TestPipeline.withSubject(indexer).withDocuments(documents).execute();
|
||||
|
||||
// Builder ref should be set to location (and only once).
|
||||
expect(lunrBuilderRefSpy).toHaveBeenCalledTimes(1);
|
||||
expect(lunrBuilderRefSpy).toHaveBeenLastCalledWith('location');
|
||||
|
||||
// Builder fields should be based on document fields.
|
||||
expect(lunrBuilderFieldSpy).toHaveBeenCalledTimes(4);
|
||||
expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('title');
|
||||
expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('text');
|
||||
expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('location');
|
||||
expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('extra');
|
||||
});
|
||||
|
||||
it('should configure lunr pipeline', async () => {
|
||||
expect(lunrBuilderSearchPipelineAddSpy).toHaveBeenLastCalledWith(
|
||||
lunr.stemmer,
|
||||
);
|
||||
expect(lunrBuilderPipelineAddSpy).toHaveBeenCalledWith(
|
||||
...[lunr.trimmer, lunr.stopWordFilter, lunr.stemmer],
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import lunr from 'lunr';
|
||||
import { BatchSearchEngineIndexer } from '../indexing';
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {
|
||||
private schemaInitialized = false;
|
||||
private builder: lunr.Builder;
|
||||
private docStore: Record<string, IndexableDocument> = {};
|
||||
|
||||
constructor() {
|
||||
super({ batchSize: 100 });
|
||||
|
||||
this.builder = new lunr.Builder();
|
||||
this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer);
|
||||
this.builder.searchPipeline.add(lunr.stemmer);
|
||||
}
|
||||
|
||||
// No async initialization required.
|
||||
async initialize(): Promise<void> {}
|
||||
async finalize(): Promise<void> {}
|
||||
|
||||
async index(documents: IndexableDocument[]): Promise<void> {
|
||||
if (!this.schemaInitialized) {
|
||||
// Make this lunr index aware of all relevant fields.
|
||||
Object.keys(documents[0]).forEach(field => {
|
||||
this.builder.field(field);
|
||||
});
|
||||
|
||||
// Set "location" field as reference field
|
||||
this.builder.ref('location');
|
||||
|
||||
this.schemaInitialized = true;
|
||||
}
|
||||
|
||||
documents.forEach(document => {
|
||||
// Add document to Lunar index
|
||||
this.builder.add(document);
|
||||
|
||||
// Store documents in memory to be able to look up document using the ref during query time
|
||||
// This is not how you should implement your SearchEngine implementation! Do not copy!
|
||||
this.docStore[document.location] = document;
|
||||
});
|
||||
}
|
||||
|
||||
buildIndex() {
|
||||
return this.builder.build();
|
||||
}
|
||||
|
||||
getDocumentStore() {
|
||||
return this.docStore;
|
||||
}
|
||||
}
|
||||
@@ -15,4 +15,8 @@
|
||||
*/
|
||||
|
||||
export { LunrSearchEngine } from './LunrSearchEngine';
|
||||
export type { ConcreteLunrQuery } from './LunrSearchEngine';
|
||||
export type {
|
||||
ConcreteLunrQuery,
|
||||
LunrQueryTranslator,
|
||||
} from './LunrSearchEngine';
|
||||
export type { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer';
|
||||
|
||||
@@ -23,6 +23,18 @@
|
||||
export { IndexBuilder } from './IndexBuilder';
|
||||
export { Scheduler } from './Scheduler';
|
||||
export { LunrSearchEngine } from './engines';
|
||||
export type {
|
||||
ConcreteLunrQuery,
|
||||
LunrQueryTranslator,
|
||||
LunrSearchEngineIndexer,
|
||||
} from './engines';
|
||||
export type {
|
||||
IndexBuilderOptions,
|
||||
RegisterCollatorParameters,
|
||||
RegisterDecoratorParameters,
|
||||
} from './types';
|
||||
export * from './indexing';
|
||||
export * from './test-utils';
|
||||
|
||||
/**
|
||||
* @deprecated Import from @backstage/search-common instead
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { BatchSearchEngineIndexer } from './BatchSearchEngineIndexer';
|
||||
import { TestPipeline } from '../test-utils';
|
||||
|
||||
const indexSpy = jest.fn().mockResolvedValue(undefined);
|
||||
const initializeSpy = jest.fn().mockResolvedValue(undefined);
|
||||
const finalizeSpy = jest.fn().mockResolvedValue(undefined);
|
||||
|
||||
class ConcreteBatchIndexer extends BatchSearchEngineIndexer {
|
||||
async index(documents: IndexableDocument[]): Promise<void> {
|
||||
return indexSpy(documents);
|
||||
}
|
||||
async initialize(): Promise<void> {
|
||||
return initializeSpy();
|
||||
}
|
||||
async finalize(): Promise<void> {
|
||||
return finalizeSpy();
|
||||
}
|
||||
}
|
||||
|
||||
describe('BatchSearchEngineIndexer', () => {
|
||||
const document = {
|
||||
title: 'Some Document',
|
||||
text: 'Some document text.',
|
||||
location: '/some/location',
|
||||
};
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
it('should work end-to-end', async () => {
|
||||
const indexer = new ConcreteBatchIndexer({ batchSize: 1 });
|
||||
await TestPipeline.withSubject(indexer)
|
||||
.withDocuments([document, document, document])
|
||||
.execute();
|
||||
expect(indexSpy).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('should call initialize at construction', () => {
|
||||
// @ts-expect-error
|
||||
const _indexer = new ConcreteBatchIndexer({ batchSize: 1 });
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Allow initialization to complete.
|
||||
setImmediate(() => {
|
||||
expect(initializeSpy).toHaveBeenCalled();
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should emit error if initialization throws', () => {
|
||||
// Cause the initializer to throw.
|
||||
const expectedError = new Error('some error');
|
||||
initializeSpy.mockRejectedValue(expectedError);
|
||||
const indexer = new ConcreteBatchIndexer({ batchSize: 1 });
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Listen for the error and assert it's what was thrown.
|
||||
indexer.on('error', error => {
|
||||
expect(error).toStrictEqual(expectedError);
|
||||
done();
|
||||
});
|
||||
|
||||
// Write a document to force the error state to become known.
|
||||
indexer.write(document);
|
||||
});
|
||||
});
|
||||
|
||||
it('should call index according to batchSize', () => {
|
||||
const indexer = new ConcreteBatchIndexer({ batchSize: 2 });
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Listen for it to finish and assert the batches.
|
||||
indexer.on('finish', () => {
|
||||
expect(indexSpy).toHaveBeenCalledTimes(2);
|
||||
expect(indexSpy).toHaveBeenNthCalledWith(1, [document, document]);
|
||||
expect(indexSpy).toHaveBeenNthCalledWith(2, [document]);
|
||||
done();
|
||||
});
|
||||
|
||||
// Write batchSize + 1 documents and end the stream.
|
||||
indexer.write(document);
|
||||
indexer.write(document);
|
||||
indexer.write(document);
|
||||
indexer.end();
|
||||
});
|
||||
});
|
||||
|
||||
it('should call index without exceeding batchSize', () => {
|
||||
const indexer = new ConcreteBatchIndexer({ batchSize: 2 });
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Listen for it to finish and assert that it still wrote.
|
||||
indexer.on('finish', () => {
|
||||
expect(indexSpy).toHaveBeenCalledTimes(1);
|
||||
expect(indexSpy).toHaveBeenNthCalledWith(1, [document]);
|
||||
done();
|
||||
});
|
||||
|
||||
// Write batchSize - 1 documents and end the stream.
|
||||
indexer.write(document);
|
||||
indexer.end();
|
||||
});
|
||||
});
|
||||
|
||||
it('should emit error if index throws', () => {
|
||||
// Cause the indexer to throw.
|
||||
const expectedError = new Error('index error');
|
||||
indexSpy.mockRejectedValue(expectedError);
|
||||
const indexer = new ConcreteBatchIndexer({ batchSize: 1 });
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Listen for the error and assert it's what was thrown.
|
||||
indexer.on('error', error => {
|
||||
expect(error).toStrictEqual(expectedError);
|
||||
done();
|
||||
});
|
||||
|
||||
indexer.write(document);
|
||||
});
|
||||
});
|
||||
|
||||
it('should emit error if finalize throws', () => {
|
||||
// Cause the indexer to throw.
|
||||
const expectedError = new Error('finalize error');
|
||||
finalizeSpy.mockRejectedValue(expectedError);
|
||||
const indexer = new ConcreteBatchIndexer({ batchSize: 1 });
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Listen for the error and assert it's what was thrown.
|
||||
indexer.on('error', error => {
|
||||
expect(error).toStrictEqual(expectedError);
|
||||
done();
|
||||
});
|
||||
|
||||
indexer.end();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { assertError } from '@backstage/errors';
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { Writable } from 'stream';
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export type BatchSearchEngineOptions = {
|
||||
batchSize: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Base class encapsulating batch-based stream processing. Useful as a base
|
||||
* class for search engine indexers.
|
||||
* @beta
|
||||
*/
|
||||
export abstract class BatchSearchEngineIndexer extends Writable {
|
||||
private batchSize: number;
|
||||
private currentBatch: IndexableDocument[] = [];
|
||||
private initialized: Promise<undefined | Error>;
|
||||
|
||||
constructor(options: BatchSearchEngineOptions) {
|
||||
super({ objectMode: true });
|
||||
this.batchSize = options.batchSize;
|
||||
|
||||
// @todo Once node v15 is minimum, convert to _construct implementation.
|
||||
this.initialized = new Promise(done => {
|
||||
// Necessary to allow concrete implementation classes to construct
|
||||
// themselves before calling their initialize() methods.
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
await this.initialize();
|
||||
done(undefined);
|
||||
} catch (e) {
|
||||
assertError(e);
|
||||
done(e);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Receives an array of indexable documents (of size this.batchSize) which
|
||||
* should be written to the search engine. This method won't be called again
|
||||
* at least until it resolves.
|
||||
*/
|
||||
public abstract index(documents: IndexableDocument[]): Promise<void>;
|
||||
|
||||
/**
|
||||
* Any asynchronous setup tasks can be performed here.
|
||||
*/
|
||||
public abstract initialize(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Any asynchronous teardown tasks can be performed here.
|
||||
*/
|
||||
public abstract finalize(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Encapsulates batch stream write logic.
|
||||
* @internal
|
||||
*/
|
||||
async _write(
|
||||
doc: IndexableDocument,
|
||||
_e: any,
|
||||
done: (error?: Error | null) => void,
|
||||
) {
|
||||
// Wait for init before proceeding. Throw error if initialization failed.
|
||||
const maybeError = await this.initialized;
|
||||
if (maybeError) {
|
||||
done(maybeError);
|
||||
return;
|
||||
}
|
||||
|
||||
this.currentBatch.push(doc);
|
||||
if (this.currentBatch.length < this.batchSize) {
|
||||
done();
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.index(this.currentBatch);
|
||||
this.currentBatch = [];
|
||||
done();
|
||||
} catch (e) {
|
||||
assertError(e);
|
||||
done(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encapsulates finalization and final error handling logic.
|
||||
* @internal
|
||||
*/
|
||||
async _final(done: (error?: Error | null) => void) {
|
||||
try {
|
||||
// Index any remaining documents.
|
||||
if (this.currentBatch.length) {
|
||||
await this.index(this.currentBatch);
|
||||
this.currentBatch = [];
|
||||
}
|
||||
await this.finalize();
|
||||
done();
|
||||
} catch (e) {
|
||||
assertError(e);
|
||||
done(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { DecoratorBase } from './DecoratorBase';
|
||||
import { TestPipeline } from '../test-utils';
|
||||
|
||||
const decorateSpy = jest.fn().mockResolvedValue(undefined);
|
||||
const initializeSpy = jest.fn().mockResolvedValue(undefined);
|
||||
const finalizeSpy = jest.fn().mockResolvedValue(undefined);
|
||||
|
||||
class ConcreteDecorator extends DecoratorBase {
|
||||
public initialize(): Promise<void> {
|
||||
return initializeSpy();
|
||||
}
|
||||
public decorate(
|
||||
document: IndexableDocument,
|
||||
): Promise<IndexableDocument | IndexableDocument[] | undefined> {
|
||||
return decorateSpy(document);
|
||||
}
|
||||
public finalize(): Promise<void> {
|
||||
return finalizeSpy();
|
||||
}
|
||||
}
|
||||
|
||||
describe('DecoratorBase', () => {
|
||||
const document = {
|
||||
title: 'Some Document',
|
||||
text: 'Some document text.',
|
||||
location: '/some/location',
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
it('should work end-to-end', async () => {
|
||||
decorateSpy.mockImplementation(doc => ({
|
||||
...doc,
|
||||
transformed: true,
|
||||
}));
|
||||
|
||||
const decorator = new ConcreteDecorator();
|
||||
const { documents } = await TestPipeline.withSubject(decorator)
|
||||
.withDocuments([document, document, document])
|
||||
.execute();
|
||||
|
||||
expect(documents.length).toBe(3);
|
||||
expect((documents[0] as unknown as any).transformed).toBe(true);
|
||||
expect((documents[1] as unknown as any).transformed).toBe(true);
|
||||
expect((documents[2] as unknown as any).transformed).toBe(true);
|
||||
});
|
||||
|
||||
it('should allow filtering', async () => {
|
||||
decorateSpy.mockResolvedValue(undefined);
|
||||
|
||||
const decorator = new ConcreteDecorator();
|
||||
const { documents } = await TestPipeline.withSubject(decorator)
|
||||
.withDocuments([document, document, document])
|
||||
.execute();
|
||||
|
||||
expect(decorateSpy).toHaveBeenCalledTimes(3);
|
||||
expect(documents.length).toBe(0);
|
||||
});
|
||||
|
||||
it('should allow fanning', async () => {
|
||||
decorateSpy.mockImplementation(doc => {
|
||||
return [doc, doc];
|
||||
});
|
||||
|
||||
const decorator = new ConcreteDecorator();
|
||||
const { documents } = await TestPipeline.withSubject(decorator)
|
||||
.withDocuments([document, document, document])
|
||||
.execute();
|
||||
|
||||
expect(decorateSpy).toHaveBeenCalledTimes(3);
|
||||
expect(documents.length).toBe(6);
|
||||
});
|
||||
|
||||
it('should call initialize at construction', () => {
|
||||
// @ts-expect-error
|
||||
const _indexer = new ConcreteDecorator();
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Allow initialization to complete.
|
||||
setImmediate(() => {
|
||||
expect(initializeSpy).toHaveBeenCalled();
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should emit error if initialization throws', () => {
|
||||
// Cause the initializer to throw.
|
||||
const expectedError = new Error('some error');
|
||||
initializeSpy.mockRejectedValue(expectedError);
|
||||
const decorator = new ConcreteDecorator();
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Listen for the error and assert it's what was thrown.
|
||||
decorator.on('error', error => {
|
||||
expect(error).toStrictEqual(expectedError);
|
||||
done();
|
||||
});
|
||||
|
||||
// Write a document to force the error state to become known.
|
||||
decorator.write(document);
|
||||
});
|
||||
});
|
||||
|
||||
it('should emit error if index throws', () => {
|
||||
// Cause the indexer to throw.
|
||||
const expectedError = new Error('decorate error');
|
||||
decorateSpy.mockRejectedValue(expectedError);
|
||||
const decorator = new ConcreteDecorator();
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Listen for the error and assert it's what was thrown.
|
||||
decorator.on('error', error => {
|
||||
expect(error).toStrictEqual(expectedError);
|
||||
done();
|
||||
});
|
||||
|
||||
decorator.write(document);
|
||||
});
|
||||
});
|
||||
|
||||
it('should emit error if finalize throws', () => {
|
||||
// Cause the indexer to throw.
|
||||
const expectedError = new Error('finalize error');
|
||||
finalizeSpy.mockRejectedValue(expectedError);
|
||||
const decorator = new ConcreteDecorator();
|
||||
|
||||
return new Promise<void>(done => {
|
||||
// Listen for the error and assert it's what was thrown.
|
||||
decorator.on('error', error => {
|
||||
expect(error).toStrictEqual(expectedError);
|
||||
done();
|
||||
});
|
||||
|
||||
decorator.end();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { assertError } from '@backstage/errors';
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { Transform } from 'stream';
|
||||
|
||||
/**
|
||||
* Base class encapsulating simple async transformations. Useful as a base
|
||||
* class for Backstage search decorators.
|
||||
* @beta
|
||||
*/
|
||||
export abstract class DecoratorBase extends Transform {
|
||||
private initialized: Promise<undefined | Error>;
|
||||
|
||||
constructor() {
|
||||
super({ objectMode: true });
|
||||
|
||||
// @todo Once node v15 is minimum, convert to _construct implementation.
|
||||
this.initialized = new Promise(done => {
|
||||
// Necessary to allow concrete implementation classes to construct
|
||||
// themselves before calling their initialize() methods.
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
await this.initialize();
|
||||
done(undefined);
|
||||
} catch (e) {
|
||||
assertError(e);
|
||||
done(e);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Any asynchronous setup tasks can be performed here.
|
||||
*/
|
||||
public abstract initialize(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Receives a single indexable document. In your decorate method, you can:
|
||||
*
|
||||
* - Resolve `undefined` to indicate the record should be omitted.
|
||||
* - Resolve a single modified document, which could contain new fields,
|
||||
* edited fields, or removed fields.
|
||||
* - Resolve an array of indexable documents, if the purpose if the decorator
|
||||
* is to convert one document into multiple derivative documents.
|
||||
*/
|
||||
public abstract decorate(
|
||||
document: IndexableDocument,
|
||||
): Promise<IndexableDocument | IndexableDocument[] | undefined>;
|
||||
|
||||
/**
|
||||
* Any asynchronous teardown tasks can be performed here.
|
||||
*/
|
||||
public abstract finalize(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Encapsulates simple transform stream logic.
|
||||
* @internal
|
||||
*/
|
||||
async _transform(
|
||||
document: IndexableDocument,
|
||||
_: any,
|
||||
done: (error?: Error | null) => void,
|
||||
) {
|
||||
// Wait for init before proceeding. Throw error if initialization failed.
|
||||
const maybeError = await this.initialized;
|
||||
if (maybeError) {
|
||||
done(maybeError);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const decorated = await this.decorate(document);
|
||||
|
||||
// If undefined was returned, omit the record and move on.
|
||||
if (decorated === undefined) {
|
||||
done();
|
||||
return;
|
||||
}
|
||||
|
||||
// If an array of documents was given, push them all.
|
||||
if (Array.isArray(decorated)) {
|
||||
decorated.forEach(doc => {
|
||||
this.push(doc);
|
||||
});
|
||||
done();
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, just push the decorated document.
|
||||
this.push(decorated);
|
||||
done();
|
||||
} catch (e) {
|
||||
assertError(e);
|
||||
done(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encapsulates finalization and final error handling logic.
|
||||
* @internal
|
||||
*/
|
||||
async _final(done: (error?: Error | null) => void) {
|
||||
try {
|
||||
await this.finalize();
|
||||
done();
|
||||
} catch (e) {
|
||||
assertError(e);
|
||||
done(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { BatchSearchEngineIndexer } from './BatchSearchEngineIndexer';
|
||||
export { DecoratorBase } from './DecoratorBase';
|
||||
export type { BatchSearchEngineOptions } from './BatchSearchEngineIndexer';
|
||||
@@ -0,0 +1,140 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { pipeline, Readable, Transform, Writable } from 'stream';
|
||||
|
||||
/**
|
||||
* Object resolved after a test pipeline is executed.
|
||||
* @beta
|
||||
*/
|
||||
export type TestPipelineResult = {
|
||||
/**
|
||||
* If an error was emitted by the pipeline, it will be set here.
|
||||
*/
|
||||
error: unknown;
|
||||
|
||||
/**
|
||||
* A list of documents collected at the end of the pipeline. If the subject
|
||||
* under test is an indexer, this will be an empty array (because your
|
||||
* indexer should have received the documents instead).
|
||||
*/
|
||||
documents: IndexableDocument[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Test utility for Backstage Search collators, decorators, and indexers.
|
||||
* @beta
|
||||
*/
|
||||
export class TestPipeline {
|
||||
private collator?: Readable;
|
||||
private decorator?: Transform;
|
||||
private indexer?: Writable;
|
||||
|
||||
private constructor({
|
||||
collator,
|
||||
decorator,
|
||||
indexer,
|
||||
}: {
|
||||
collator?: Readable;
|
||||
decorator?: Transform;
|
||||
indexer?: Writable;
|
||||
}) {
|
||||
this.collator = collator;
|
||||
this.decorator = decorator;
|
||||
this.indexer = indexer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide the collator, decorator, or indexer to be tested.
|
||||
*/
|
||||
static withSubject(subject: Readable | Transform | Writable) {
|
||||
if (subject instanceof Transform) {
|
||||
return new TestPipeline({ decorator: subject });
|
||||
}
|
||||
|
||||
if (subject instanceof Readable) {
|
||||
return new TestPipeline({ collator: subject });
|
||||
}
|
||||
|
||||
if (subject instanceof Writable) {
|
||||
return new TestPipeline({ indexer: subject });
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
'Unknown test subject: are you passing a readable, writable, or transform stream?',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide documents for testing decorators and indexers.
|
||||
*/
|
||||
withDocuments(documents: IndexableDocument[]): TestPipeline {
|
||||
if (this.collator) {
|
||||
throw new Error('Cannot provide documents when testing a collator.');
|
||||
}
|
||||
|
||||
// Set a naive readable stream that just pushes all given documents.
|
||||
this.collator = new Readable({ objectMode: true });
|
||||
this.collator._read = () => {};
|
||||
process.nextTick(() => {
|
||||
documents.forEach(document => {
|
||||
this.collator!.push(document);
|
||||
});
|
||||
this.collator!.push(null);
|
||||
});
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the test pipeline so that you can make assertions about the result
|
||||
* or behavior of the given test subject.
|
||||
*/
|
||||
async execute(): Promise<TestPipelineResult> {
|
||||
const documents: IndexableDocument[] = [];
|
||||
if (!this.collator) {
|
||||
throw new Error(
|
||||
'Cannot execute pipeline without a collator or documents',
|
||||
);
|
||||
}
|
||||
|
||||
// If we are here and there is no indexer, we are testing a collator or a
|
||||
// decorator. Set up a naive writable that captures documents in memory.
|
||||
if (!this.indexer) {
|
||||
this.indexer = new Writable({ objectMode: true });
|
||||
this.indexer._write = (document: IndexableDocument, _, done) => {
|
||||
documents.push(document);
|
||||
done();
|
||||
};
|
||||
}
|
||||
|
||||
return new Promise<TestPipelineResult>(done => {
|
||||
const pipes: (Readable | Transform | Writable)[] = [this.collator!];
|
||||
if (this.decorator) {
|
||||
pipes.push(this.decorator);
|
||||
}
|
||||
pipes.push(this.indexer!);
|
||||
|
||||
pipeline(pipes, error => {
|
||||
done({
|
||||
error,
|
||||
documents,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { TestPipeline } from './TestPipeline';
|
||||
export type { TestPipelineResult } from './TestPipeline';
|
||||
@@ -14,10 +14,24 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { DocumentCollator, DocumentDecorator } from '@backstage/search-common';
|
||||
import {
|
||||
DocumentCollatorFactory,
|
||||
DocumentDecoratorFactory,
|
||||
SearchEngine,
|
||||
} from '@backstage/search-common';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
/**
|
||||
* @beta
|
||||
*/
|
||||
export type IndexBuilderOptions = {
|
||||
searchEngine: SearchEngine;
|
||||
logger: Logger;
|
||||
};
|
||||
|
||||
/**
|
||||
* Parameters required to register a collator.
|
||||
* @beta
|
||||
*/
|
||||
export interface RegisterCollatorParameters {
|
||||
/**
|
||||
@@ -26,17 +40,18 @@ export interface RegisterCollatorParameters {
|
||||
defaultRefreshIntervalSeconds: number;
|
||||
|
||||
/**
|
||||
* The collator class responsible for returning all documents of the given type.
|
||||
* The class responsible for returning the document collator of the given type.
|
||||
*/
|
||||
collator: DocumentCollator;
|
||||
factory: DocumentCollatorFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parameters required to register a decorator
|
||||
* @beta
|
||||
*/
|
||||
export interface RegisterDecoratorParameters {
|
||||
/**
|
||||
* The decorator class responsible for appending or modifying documents of the given type(s).
|
||||
* The class responsible for returning the decorator which appends, modifies, or filters documents.
|
||||
*/
|
||||
decorator: DocumentDecorator;
|
||||
factory: DocumentDecoratorFactory;
|
||||
}
|
||||
|
||||
@@ -62,7 +62,7 @@ describe('AuthorizedSearchEngine', () => {
|
||||
setTranslator: () => {
|
||||
throw new Error('Function not implemented. 1');
|
||||
},
|
||||
index: () => {
|
||||
getIndexer: () => {
|
||||
throw new Error('Function not implemented.2');
|
||||
},
|
||||
query: mockedQuery,
|
||||
|
||||
@@ -25,7 +25,6 @@ import {
|
||||
} from '@backstage/plugin-permission-common';
|
||||
import {
|
||||
DocumentTypeInfo,
|
||||
IndexableDocument,
|
||||
QueryRequestOptions,
|
||||
QueryTranslator,
|
||||
SearchEngine,
|
||||
@@ -35,6 +34,7 @@ import {
|
||||
} from '@backstage/search-common';
|
||||
import { Config } from '@backstage/config';
|
||||
import { InputError } from '@backstage/errors';
|
||||
import { Writable } from 'stream';
|
||||
|
||||
export function decodePageCursor(pageCursor?: string): { page: number } {
|
||||
if (!pageCursor) {
|
||||
@@ -78,8 +78,8 @@ export class AuthorizedSearchEngine implements SearchEngine {
|
||||
this.searchEngine.setTranslator(translator);
|
||||
}
|
||||
|
||||
async index(type: string, documents: IndexableDocument[]): Promise<void> {
|
||||
this.searchEngine.index(type, documents);
|
||||
async getIndexer(type: string): Promise<Writable> {
|
||||
return this.searchEngine.getIndexer(type);
|
||||
}
|
||||
|
||||
async query(
|
||||
|
||||
@@ -105,7 +105,7 @@ describe('createRouter', () => {
|
||||
beforeAll(async () => {
|
||||
const logger = getVoidLogger();
|
||||
mockSearchEngine = {
|
||||
index: jest.fn(),
|
||||
getIndexer: jest.fn(),
|
||||
setTranslator: jest.fn(),
|
||||
query: jest.fn(),
|
||||
};
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
/// <reference types="node" />
|
||||
|
||||
import { CatalogApi } from '@backstage/catalog-client';
|
||||
import { Config } from '@backstage/config';
|
||||
import { DocumentCollator } from '@backstage/search-common';
|
||||
import { DocumentCollatorFactory } from '@backstage/search-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import express from 'express';
|
||||
import { GeneratorBuilder } from '@backstage/techdocs-common';
|
||||
@@ -16,14 +18,15 @@ import { PluginCacheManager } from '@backstage/backend-common';
|
||||
import { PluginEndpointDiscovery } from '@backstage/backend-common';
|
||||
import { PreparerBuilder } from '@backstage/techdocs-common';
|
||||
import { PublisherBase } from '@backstage/techdocs-common';
|
||||
import { Readable } from 'stream';
|
||||
import { TechDocsDocument } from '@backstage/techdocs-common';
|
||||
import { TokenManager } from '@backstage/backend-common';
|
||||
|
||||
// @public
|
||||
export function createRouter(options: RouterOptions): Promise<express.Router>;
|
||||
|
||||
// @public
|
||||
export class DefaultTechDocsCollator implements DocumentCollator {
|
||||
// @public @deprecated
|
||||
export class DefaultTechDocsCollator {
|
||||
// (undocumented)
|
||||
protected applyArgsToFormat(
|
||||
format: string,
|
||||
@@ -42,6 +45,21 @@ export class DefaultTechDocsCollator implements DocumentCollator {
|
||||
readonly visibilityPermission: Permission;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
|
||||
// (undocumented)
|
||||
static fromConfig(
|
||||
config: Config,
|
||||
options: TechDocsCollatorFactoryOptions,
|
||||
): DefaultTechDocsCollatorFactory;
|
||||
// (undocumented)
|
||||
getCollator(): Promise<Readable>;
|
||||
// (undocumented)
|
||||
readonly type: string;
|
||||
// (undocumented)
|
||||
readonly visibilityPermission: Permission;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface DocsBuildStrategy {
|
||||
// (undocumented)
|
||||
@@ -81,6 +99,17 @@ export type ShouldBuildParameters = {
|
||||
entity: Entity;
|
||||
};
|
||||
|
||||
// @public
|
||||
export type TechDocsCollatorFactoryOptions = {
|
||||
discovery: PluginEndpointDiscovery;
|
||||
logger: Logger_2;
|
||||
tokenManager: TokenManager;
|
||||
locationTemplate?: string;
|
||||
catalogClient?: CatalogApi;
|
||||
parallelismLimit?: number;
|
||||
legacyPathCasing?: boolean;
|
||||
};
|
||||
|
||||
// @public
|
||||
export type TechDocsCollatorOptions = {
|
||||
discovery: PluginEndpointDiscovery;
|
||||
|
||||
@@ -56,6 +56,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/cli": "^0.14.1",
|
||||
"@backstage/plugin-search-backend-node": "0.4.7",
|
||||
"@backstage/test-utils": "^0.2.6",
|
||||
"@types/dockerode": "^3.3.0",
|
||||
"msw": "^0.35.0",
|
||||
|
||||
@@ -29,8 +29,14 @@ export type {
|
||||
ShouldBuildParameters,
|
||||
} from './service';
|
||||
|
||||
export { DefaultTechDocsCollator } from './search';
|
||||
export type { TechDocsCollatorOptions } from './search';
|
||||
export {
|
||||
DefaultTechDocsCollator,
|
||||
DefaultTechDocsCollatorFactory,
|
||||
} from './search';
|
||||
export type {
|
||||
TechDocsCollatorFactoryOptions,
|
||||
TechDocsCollatorOptions,
|
||||
} from './search';
|
||||
|
||||
/**
|
||||
* @deprecated Use directly from @backstage/techdocs-common
|
||||
|
||||
@@ -72,18 +72,6 @@ const expectedEntities: Entity[] = [
|
||||
owner: 'someone',
|
||||
},
|
||||
},
|
||||
{
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: {
|
||||
name: 'test-entity',
|
||||
description: 'The expected description',
|
||||
},
|
||||
spec: {
|
||||
type: 'some-type',
|
||||
lifecycle: 'experimental',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('DefaultTechDocsCollator with legacyPathCasing configuration', () => {
|
||||
|
||||
@@ -24,7 +24,6 @@ import {
|
||||
RELATION_OWNED_BY,
|
||||
stringifyEntityRef,
|
||||
} from '@backstage/catalog-model';
|
||||
import { DocumentCollator } from '@backstage/search-common';
|
||||
import fetch from 'node-fetch';
|
||||
import unescape from 'lodash/unescape';
|
||||
import { Logger } from 'winston';
|
||||
@@ -69,8 +68,10 @@ type EntityInfo = {
|
||||
* A search collator responsible for gathering and transforming TechDocs documents.
|
||||
*
|
||||
* @public
|
||||
* @deprecated Upgrade to a more recent `@backstage/search-backend-node` and
|
||||
* use `DefaultTechDocsCollatorFactory` instead.
|
||||
*/
|
||||
export class DefaultTechDocsCollator implements DocumentCollator {
|
||||
export class DefaultTechDocsCollator {
|
||||
public readonly type: string = 'techdocs';
|
||||
public readonly visibilityPermission = catalogEntityReadPermission;
|
||||
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import {
|
||||
getVoidLogger,
|
||||
PluginEndpointDiscovery,
|
||||
TokenManager,
|
||||
} from '@backstage/backend-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { TestPipeline } from '@backstage/plugin-search-backend-node';
|
||||
import { setupRequestMockHandlers } from '@backstage/test-utils';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { Readable } from 'stream';
|
||||
import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
const mockSearchDocIndex = {
|
||||
config: {
|
||||
lang: ['en'],
|
||||
min_search_length: 3,
|
||||
prebuild_index: false,
|
||||
separator: '[\\s\\-]+',
|
||||
},
|
||||
docs: [
|
||||
{
|
||||
location: '',
|
||||
text: 'docs docs docs',
|
||||
title: 'Home',
|
||||
},
|
||||
{
|
||||
location: 'local-development/',
|
||||
text: 'Docs for first subtitle',
|
||||
title: 'Local development',
|
||||
},
|
||||
{
|
||||
location: 'local-development/#development',
|
||||
text: 'Docs for sub-subtitle',
|
||||
title: 'Development',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const expectedEntities: Entity[] = [
|
||||
{
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: {
|
||||
title: 'Test Entity with Docs!',
|
||||
name: 'test-entity-with-docs',
|
||||
description: 'Documented description',
|
||||
annotations: {
|
||||
'backstage.io/techdocs-ref': './',
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
type: 'dog',
|
||||
lifecycle: 'experimental',
|
||||
owner: 'someone',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('DefaultTechDocsCollatorFactory', () => {
|
||||
const config = new ConfigReader({});
|
||||
const mockDiscoveryApi: jest.Mocked<PluginEndpointDiscovery> = {
|
||||
getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'),
|
||||
getExternalBaseUrl: jest.fn(),
|
||||
};
|
||||
const mockTokenManager: jest.Mocked<TokenManager> = {
|
||||
getToken: jest.fn().mockResolvedValue({ token: '' }),
|
||||
authenticate: jest.fn(),
|
||||
};
|
||||
const options = {
|
||||
discovery: mockDiscoveryApi,
|
||||
logger: getVoidLogger(),
|
||||
tokenManager: mockTokenManager,
|
||||
};
|
||||
|
||||
it('has expected type', () => {
|
||||
const factory = DefaultTechDocsCollatorFactory.fromConfig(config, options);
|
||||
expect(factory.type).toBe('techdocs');
|
||||
});
|
||||
|
||||
describe('getCollator', () => {
|
||||
let factory: DefaultTechDocsCollatorFactory;
|
||||
let collator: Readable;
|
||||
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
beforeEach(async () => {
|
||||
factory = DefaultTechDocsCollatorFactory.fromConfig(config, options);
|
||||
collator = await factory.getCollator();
|
||||
|
||||
worker.use(
|
||||
rest.get(
|
||||
'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json',
|
||||
(_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)),
|
||||
),
|
||||
rest.get('http://test-backend/entities', (req, res, ctx) => {
|
||||
// Imitate offset/limit pagination.
|
||||
const offset = parseInt(
|
||||
req.url.searchParams.get('offset') || '0',
|
||||
10,
|
||||
);
|
||||
const limit = parseInt(
|
||||
req.url.searchParams.get('limit') || '500',
|
||||
10,
|
||||
);
|
||||
|
||||
// Limit 50 corresponds to a case testing pagination.
|
||||
if (limit === 50) {
|
||||
// Return 50 copies of invalid entities on the first request.
|
||||
if (offset === 0) {
|
||||
return res(ctx.status(200), ctx.json(Array(50).fill({})));
|
||||
}
|
||||
// Then just the regular 2 on the second.
|
||||
return res(ctx.status(200), ctx.json(expectedEntities));
|
||||
}
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json(expectedEntities.slice(offset, limit + offset)),
|
||||
);
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns a readable stream', async () => {
|
||||
expect(collator).toBeInstanceOf(Readable);
|
||||
});
|
||||
|
||||
it('fetches from the configured catalog and tech docs services', async () => {
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog');
|
||||
expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs');
|
||||
expect(documents).toHaveLength(mockSearchDocIndex.docs.length);
|
||||
});
|
||||
|
||||
it('should create documents for each tech docs search index', async () => {
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
const entity = expectedEntities[0];
|
||||
documents.forEach((document, idx) => {
|
||||
expect(document).toMatchObject({
|
||||
title: mockSearchDocIndex.docs[idx].title,
|
||||
location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
|
||||
text: mockSearchDocIndex.docs[idx].text,
|
||||
namespace: 'default',
|
||||
entityTitle: entity!.metadata.title,
|
||||
componentType: entity!.spec!.type,
|
||||
lifecycle: entity!.spec!.lifecycle,
|
||||
owner: '',
|
||||
kind: entity.kind.toLocaleLowerCase('en-US'),
|
||||
name: entity.metadata.name,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('maps a returned entity with a custom locationTemplate', async () => {
|
||||
// Provide an alternate location template.
|
||||
factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
|
||||
discovery: mockDiscoveryApi,
|
||||
tokenManager: mockTokenManager,
|
||||
locationTemplate: '/software/:name',
|
||||
logger,
|
||||
});
|
||||
collator = await factory.getCollator();
|
||||
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
|
||||
expect(documents[0]).toMatchObject({
|
||||
location: '/software/test-entity-with-docs',
|
||||
});
|
||||
});
|
||||
|
||||
it('paginates through catalog entities using batchSize', async () => {
|
||||
// A parallelismLimit of 1 is a catalog limit of 50 per request. Code
|
||||
// above in the /entities handler ensures valid entities are only
|
||||
// returned on the second page.
|
||||
factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
|
||||
...options,
|
||||
parallelismLimit: 1,
|
||||
});
|
||||
collator = await factory.getCollator();
|
||||
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
|
||||
// Only 1 entity with TechDocs configured multipled by 3 pages.
|
||||
expect(documents).toHaveLength(3);
|
||||
});
|
||||
|
||||
describe('with legacyPathCasing configuration', () => {
|
||||
beforeEach(async () => {
|
||||
const legacyConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
legacyUseCaseSensitiveTripletPaths: true,
|
||||
},
|
||||
});
|
||||
factory = DefaultTechDocsCollatorFactory.fromConfig(
|
||||
legacyConfig,
|
||||
options,
|
||||
);
|
||||
collator = await factory.getCollator();
|
||||
});
|
||||
|
||||
it('should create documents for each tech docs search index', async () => {
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
const entity = expectedEntities[0];
|
||||
documents.forEach((document, idx) => {
|
||||
expect(document).toMatchObject({
|
||||
title: mockSearchDocIndex.docs[idx].title,
|
||||
location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
|
||||
text: mockSearchDocIndex.docs[idx].text,
|
||||
namespace: 'default',
|
||||
entityTitle: entity!.metadata.title,
|
||||
componentType: entity!.spec!.type,
|
||||
lifecycle: entity!.spec!.lifecycle,
|
||||
owner: '',
|
||||
kind: entity.kind,
|
||||
name: entity.metadata.name,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,254 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
PluginEndpointDiscovery,
|
||||
TokenManager,
|
||||
} from '@backstage/backend-common';
|
||||
import {
|
||||
CatalogApi,
|
||||
CatalogClient,
|
||||
CATALOG_FILTER_EXISTS,
|
||||
} from '@backstage/catalog-client';
|
||||
import {
|
||||
Entity,
|
||||
parseEntityRef,
|
||||
RELATION_OWNED_BY,
|
||||
stringifyEntityRef,
|
||||
} from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common';
|
||||
import { DocumentCollatorFactory } from '@backstage/search-common';
|
||||
import { TechDocsDocument } from '@backstage/techdocs-common';
|
||||
import unescape from 'lodash/unescape';
|
||||
import fetch from 'node-fetch';
|
||||
import pLimit from 'p-limit';
|
||||
import { Readable } from 'stream';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
interface MkSearchIndexDoc {
|
||||
title: string;
|
||||
text: string;
|
||||
location: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options to configure the TechDocs collator factory
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type TechDocsCollatorFactoryOptions = {
|
||||
discovery: PluginEndpointDiscovery;
|
||||
logger: Logger;
|
||||
tokenManager: TokenManager;
|
||||
locationTemplate?: string;
|
||||
catalogClient?: CatalogApi;
|
||||
parallelismLimit?: number;
|
||||
legacyPathCasing?: boolean;
|
||||
};
|
||||
|
||||
type EntityInfo = {
|
||||
name: string;
|
||||
namespace: string;
|
||||
kind: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A search collator factory responsible for gathering and transforming
|
||||
* TechDocs documents.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
|
||||
public readonly type: string = 'techdocs';
|
||||
public readonly visibilityPermission = catalogEntityReadPermission;
|
||||
|
||||
private discovery: PluginEndpointDiscovery;
|
||||
private locationTemplate: string;
|
||||
private readonly logger: Logger;
|
||||
private readonly catalogClient: CatalogApi;
|
||||
private readonly tokenManager: TokenManager;
|
||||
private readonly parallelismLimit: number;
|
||||
private readonly legacyPathCasing: boolean;
|
||||
|
||||
private constructor(options: TechDocsCollatorFactoryOptions) {
|
||||
this.discovery = options.discovery;
|
||||
this.locationTemplate =
|
||||
options.locationTemplate || '/docs/:namespace/:kind/:name/:path';
|
||||
this.logger = options.logger;
|
||||
this.catalogClient =
|
||||
options.catalogClient ||
|
||||
new CatalogClient({ discoveryApi: options.discovery });
|
||||
this.parallelismLimit = options.parallelismLimit ?? 10;
|
||||
this.legacyPathCasing = options.legacyPathCasing ?? false;
|
||||
this.tokenManager = options.tokenManager;
|
||||
}
|
||||
|
||||
static fromConfig(config: Config, options: TechDocsCollatorFactoryOptions) {
|
||||
const legacyPathCasing =
|
||||
config.getOptionalBoolean(
|
||||
'techdocs.legacyUseCaseSensitiveTripletPaths',
|
||||
) || false;
|
||||
return new DefaultTechDocsCollatorFactory({ ...options, legacyPathCasing });
|
||||
}
|
||||
|
||||
async getCollator(): Promise<Readable> {
|
||||
return Readable.from(this.execute());
|
||||
}
|
||||
|
||||
private async *execute(): AsyncGenerator<TechDocsDocument, void, undefined> {
|
||||
const limit = pLimit(this.parallelismLimit);
|
||||
const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs');
|
||||
const { token } = await this.tokenManager.getToken();
|
||||
let entitiesRetrieved = 0;
|
||||
let moreEntitiesToGet = true;
|
||||
|
||||
// Offset/limit pagination is used on the Catalog Client in order to
|
||||
// limit (and allow some control over) memory used by the search backend
|
||||
// at index-time. The batchSize is calculated as a factor of the given
|
||||
// parallelism limit to simplify configuration.
|
||||
const batchSize = this.parallelismLimit * 50;
|
||||
while (moreEntitiesToGet) {
|
||||
const entities = (
|
||||
await this.catalogClient.getEntities(
|
||||
{
|
||||
filter: {
|
||||
'metadata.annotations.backstage.io/techdocs-ref':
|
||||
CATALOG_FILTER_EXISTS,
|
||||
},
|
||||
fields: [
|
||||
'kind',
|
||||
'namespace',
|
||||
'metadata.annotations',
|
||||
'metadata.name',
|
||||
'metadata.title',
|
||||
'metadata.namespace',
|
||||
'spec.type',
|
||||
'spec.lifecycle',
|
||||
'relations',
|
||||
],
|
||||
limit: batchSize,
|
||||
offset: entitiesRetrieved,
|
||||
},
|
||||
{ token },
|
||||
)
|
||||
).items;
|
||||
|
||||
// Control looping through entity batches.
|
||||
moreEntitiesToGet = entities.length === batchSize;
|
||||
entitiesRetrieved += entities.length;
|
||||
|
||||
const docPromises = entities
|
||||
.filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref'])
|
||||
.map((entity: Entity) =>
|
||||
limit(async (): Promise<TechDocsDocument[]> => {
|
||||
const entityInfo =
|
||||
DefaultTechDocsCollatorFactory.handleEntityInfoCasing(
|
||||
this.legacyPathCasing,
|
||||
{
|
||||
kind: entity.kind,
|
||||
namespace: entity.metadata.namespace || 'default',
|
||||
name: entity.metadata.name,
|
||||
},
|
||||
);
|
||||
|
||||
try {
|
||||
const searchIndexResponse = await fetch(
|
||||
DefaultTechDocsCollatorFactory.constructDocsIndexUrl(
|
||||
techDocsBaseUrl,
|
||||
entityInfo,
|
||||
),
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
const searchIndex = await searchIndexResponse.json();
|
||||
|
||||
return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
|
||||
title: unescape(doc.title),
|
||||
text: unescape(doc.text || ''),
|
||||
location: this.applyArgsToFormat(
|
||||
this.locationTemplate || '/docs/:namespace/:kind/:name/:path',
|
||||
{
|
||||
...entityInfo,
|
||||
path: doc.location,
|
||||
},
|
||||
),
|
||||
path: doc.location,
|
||||
...entityInfo,
|
||||
entityTitle: entity.metadata.title,
|
||||
componentType: entity.spec?.type?.toString() || 'other',
|
||||
lifecycle: (entity.spec?.lifecycle as string) || '',
|
||||
owner: getSimpleEntityOwnerString(entity),
|
||||
authorization: {
|
||||
resourceRef: stringifyEntityRef(entity),
|
||||
},
|
||||
}));
|
||||
} catch (e) {
|
||||
this.logger.debug(
|
||||
`Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
|
||||
e,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}),
|
||||
);
|
||||
yield* (await Promise.all(docPromises)).flat();
|
||||
}
|
||||
}
|
||||
|
||||
private applyArgsToFormat(
|
||||
format: string,
|
||||
args: Record<string, string>,
|
||||
): string {
|
||||
let formatted = format;
|
||||
for (const [key, value] of Object.entries(args)) {
|
||||
formatted = formatted.replace(`:${key}`, value);
|
||||
}
|
||||
return formatted;
|
||||
}
|
||||
|
||||
private static constructDocsIndexUrl(
|
||||
techDocsBaseUrl: string,
|
||||
entityInfo: { kind: string; namespace: string; name: string },
|
||||
) {
|
||||
return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`;
|
||||
}
|
||||
|
||||
private static handleEntityInfoCasing(
|
||||
legacyPaths: boolean,
|
||||
entityInfo: EntityInfo,
|
||||
): EntityInfo {
|
||||
return legacyPaths
|
||||
? entityInfo
|
||||
: Object.entries(entityInfo).reduce((acc, [key, value]) => {
|
||||
return { ...acc, [key]: value.toLocaleLowerCase('en-US') };
|
||||
}, {} as EntityInfo);
|
||||
}
|
||||
}
|
||||
|
||||
function getSimpleEntityOwnerString(entity: Entity): string {
|
||||
if (entity.relations) {
|
||||
const owner = entity.relations.find(r => r.type === RELATION_OWNED_BY);
|
||||
if (owner) {
|
||||
const { name } = parseEntityRef(owner.targetRef);
|
||||
return name;
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
@@ -13,6 +13,12 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
|
||||
|
||||
export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
|
||||
export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory';
|
||||
|
||||
/**
|
||||
* todo(backstage/techdocs-core): stop exporting these in a future release.
|
||||
*/
|
||||
export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
|
||||
export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator';
|
||||
|
||||
@@ -213,6 +213,7 @@ const NO_WARNING_PACKAGES = [
|
||||
'packages/errors',
|
||||
'packages/integration',
|
||||
'packages/integration-react',
|
||||
'packages/search-common',
|
||||
'packages/techdocs-common',
|
||||
'packages/test-utils',
|
||||
'packages/theme',
|
||||
@@ -235,6 +236,7 @@ const NO_WARNING_PACKAGES = [
|
||||
'plugins/scaffolder-backend-module-rails',
|
||||
'plugins/scaffolder-backend-module-yeoman',
|
||||
'plugins/scaffolder-common',
|
||||
'plugins/search-backend-node',
|
||||
'plugins/techdocs-backend',
|
||||
'plugins/tech-insights',
|
||||
'plugins/tech-insights-backend',
|
||||
|
||||
Reference in New Issue
Block a user