diff --git a/.changeset/brave-carrots-occur.md b/.changeset/brave-carrots-occur.md new file mode 100644 index 0000000000..017a9bdd8c --- /dev/null +++ b/.changeset/brave-carrots-occur.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-pg': patch +--- + +Added `indexerBatchSize` option to be able to control the size of the batches being indexed. Also added a debug log entry to list out all the entities in the batch diff --git a/packages/backend/package.json b/packages/backend/package.json index 6d50cd0f28..e6ca993699 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -65,8 +65,11 @@ "@backstage/plugin-scaffolder-backend-module-confluence-to-markdown": "workspace:^", "@backstage/plugin-scaffolder-backend-module-rails": "workspace:^", "@backstage/plugin-search-backend": "workspace:^", + "@backstage/plugin-search-backend-module-catalog": "workspace:^", "@backstage/plugin-search-backend-module-elasticsearch": "workspace:^", + "@backstage/plugin-search-backend-module-explore": "workspace:^", "@backstage/plugin-search-backend-module-pg": "workspace:^", + "@backstage/plugin-search-backend-module-techdocs": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/plugin-tech-insights-backend": "workspace:^", diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 4290fcd68a..7babfe7e1f 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -16,8 +16,8 @@ import { useHotCleanup } from '@backstage/backend-common'; import { DefaultAdrCollatorFactory } from '@backstage/plugin-adr-backend'; -import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; -import { ToolDocumentCollatorFactory } from '@backstage/plugin-explore-backend'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-search-backend-module-catalog'; +import { ToolDocumentCollatorFactory } from '@backstage/plugin-search-backend-module-explore'; import { createRouter } from '@backstage/plugin-search-backend'; import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg'; @@ -26,7 +26,7 @@ import { LunrSearchEngine, } from '@backstage/plugin-search-backend-node'; import { SearchEngine } from '@backstage/plugin-search-common'; -import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-search-backend-module-techdocs'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; @@ -43,6 +43,7 @@ async function createSearchEngine( if (await PgSearchEngine.supported(env.database)) { return await PgSearchEngine.fromConfig(env.config, { database: env.database, + logger: env.logger, }); } diff --git a/plugins/search-backend-module-pg/config.d.ts b/plugins/search-backend-module-pg/config.d.ts index 7367b770bc..3039cd1d28 100644 --- a/plugins/search-backend-module-pg/config.d.ts +++ b/plugins/search-backend-module-pg/config.d.ts @@ -57,6 +57,10 @@ export interface Config { */ fragmentDelimiter?: string; }; + /** + * Batch size to use when indexing + */ + indexerBatchSize?: number; }; }; } diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index d2cf80e0ca..1a27b9f981 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -86,6 +86,7 @@ export type PgSearchHighlightOptions = { export class PgSearchEngine implements SearchEngine { private readonly logger?: Logger; private readonly highlightOptions: PgSearchHighlightOptions; + private readonly indexerBatchSize: number; /** * @deprecated This will be marked as private in a future release, please us fromConfig instead @@ -114,6 +115,8 @@ export class PgSearchEngine implements SearchEngine { highlightConfig?.getOptionalString('fragmentDelimiter') ?? ' ... ', }; this.highlightOptions = highlightOptions; + this.indexerBatchSize = + config.getOptionalNumber('search.pg.indexerBatchSize') ?? 1000; this.logger = logger; } @@ -178,7 +181,7 @@ export class PgSearchEngine implements SearchEngine { async getIndexer(type: string) { return new PgSearchEngineIndexer({ - batchSize: 1000, + batchSize: this.indexerBatchSize, type, databaseStore: this.databaseStore, logger: this.logger?.child({ documentType: type }), diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts index 47ea812a92..9fe0e7091c 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts @@ -59,6 +59,11 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { async index(documents: IndexableDocument[]): Promise { this.numRecords += documents.length; + const refs = [...new Set(documents.map(d => d.authorization?.resourceRef))]; + this.logger.debug( + `Attempting to index the following entities: ${refs.toString()}`, + ); + try { await this.store.insertDocuments(this.tx!, this.type, documents); } catch (e) { diff --git a/yarn.lock b/yarn.lock index 873727c402..015913d8b8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -25812,8 +25812,11 @@ __metadata: "@backstage/plugin-scaffolder-backend-module-confluence-to-markdown": "workspace:^" "@backstage/plugin-scaffolder-backend-module-rails": "workspace:^" "@backstage/plugin-search-backend": "workspace:^" + "@backstage/plugin-search-backend-module-catalog": "workspace:^" "@backstage/plugin-search-backend-module-elasticsearch": "workspace:^" + "@backstage/plugin-search-backend-module-explore": "workspace:^" "@backstage/plugin-search-backend-module-pg": "workspace:^" + "@backstage/plugin-search-backend-module-techdocs": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/plugin-tech-insights-backend": "workspace:^"