From 49fe8c20005c8a09780a044f917bf388b096ff33 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Tue, 28 Feb 2023 14:34:34 +0100 Subject: [PATCH 01/27] define interfaces for extension points and services Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- .../search-backend-node/src/IndexBuilder.ts | 7 ++- plugins/search-backend-node/src/index.ts | 2 +- plugins/search-backend-node/src/types.ts | 26 ++++++++ plugins/search-backend/src/plugin.ts | 59 +++++++++++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 plugins/search-backend/src/plugin.ts diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts index 907519f289..9397c098aa 100644 --- a/plugins/search-backend-node/src/IndexBuilder.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { createServiceRef } from '@backstage/backend-plugin-api'; import { DocumentDecoratorFactory, DocumentTypeInfo, @@ -23,16 +24,20 @@ import { Transform, pipeline } from 'stream'; import { Logger } from 'winston'; import { Scheduler } from './Scheduler'; import { + IndexBuilderService, IndexBuilderOptions, RegisterCollatorParameters, RegisterDecoratorParameters, } from './types'; +export const searchIndexBuilderRef = createServiceRef({ + id: 'search.index-node', +}); /** * Used for adding collators, decorators and compile them into tasks which are added to a scheduler returned to the caller. * @public */ -export class IndexBuilder { +export class IndexBuilder implements IndexBuilderService { private collators: Record; private decorators: Record; private documentTypes: Record; diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index a188509c1c..1bcd31de93 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -20,7 +20,7 @@ * @packageDocumentation */ -export { IndexBuilder } from './IndexBuilder'; +export { IndexBuilder, searchIndexBuilderRef } from './IndexBuilder'; export { Scheduler } from './Scheduler'; export * from './collators'; export { LunrSearchEngine } from './engines'; diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index dfcf4d12ce..94b21cceec 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -18,9 +18,11 @@ import { TaskRunner } from '@backstage/backend-tasks'; import { DocumentCollatorFactory, DocumentDecoratorFactory, + DocumentTypeInfo, SearchEngine, } from '@backstage/plugin-search-common'; import { Logger } from 'winston'; +import { Scheduler } from './Scheduler'; /** * Options required to instantiate the index builder. @@ -57,3 +59,27 @@ export interface RegisterDecoratorParameters { */ factory: DocumentDecoratorFactory; } + +export type IndexBuilderServiceBuildOptions = { + searchEngine: SearchEngine; + collators: RegisterCollatorParameters[]; + decorators: RegisterDecoratorParameters[]; +}; +export interface IndexBuilderService { + build(options: IndexBuilderServiceBuildOptions): Promise<{ + scheduler: Scheduler; + }>; + getDocumentTypes(): Record; +} + +export interface SearchIndexRegistryExtensionPoint { + addCollator(options: RegisterCollatorParameters): void; + addDecorator(options: RegisterDecoratorParameters): void; + getCollators(): RegisterCollatorParameters[]; + getDecorators(): RegisterDecoratorParameters[]; +} + +export interface SearchEngineRegistryExtensionPoint { + setSearchEngine(searchEngine: SearchEngine): void; + getSearchEngine(): SearchEngine; +} diff --git a/plugins/search-backend/src/plugin.ts b/plugins/search-backend/src/plugin.ts new file mode 100644 index 0000000000..5d7c7935a8 --- /dev/null +++ b/plugins/search-backend/src/plugin.ts @@ -0,0 +1,59 @@ +import { + coreServices, + createBackendPlugin, +} from '@backstage/backend-plugin-api'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + searchIndexBuilderRef, + searchEngineExtensionPoint, + searchIndexExtensionPoint, +} from '@backstage/plugin-search-backend-node'; + +import { createRouter } from './service/router'; + +export const searchPlugin = createBackendPlugin({ + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + logger: coreServices.logger, + config: coreServices.config, + permissions: coreServices.permissions, + http: coreServices.httpRouter, + searchIndexBuilder: searchIndexBuilderRef, + searchEngineRegistry: searchEngineExtensionPoint, + searchIndexRegistry: searchIndexExtensionPoint, + }, + async init({ + config, + logger, + permissions, + http, + searchEngineRegistry, + searchIndexRegistry, + searchIndexBuilder, + }) { + const searchEngine = searchEngineRegistry.getSearchEngine(); + const collators = searchIndexRegistry.getCollators(); + const decorators = searchIndexRegistry.getDecorators(); + + const { scheduler } = searchIndexBuilder.build({ + searchEngine, + collators, + decorators, + }); + scheduler.start(); + + const router = await createRouter({ + config, + permissions, + logger: loggerToWinstonLogger(logger), + engine: searchEngine, + types: searchIndexBuilder.getDocumentTypes(), + }); + // We register the router with the http service. + http.use(router); + }, + }); + }, +}); From b49c312875a08b24f8633815cffee99b82fcea7d Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 1 Mar 2023 21:14:18 +0100 Subject: [PATCH 02/27] feat(search): create search services and extension points Signed-off-by: Camila Belo --- plugins/search-backend-node/package.json | 1 + .../search-backend-node/src/IndexBuilder.ts | 7 +- plugins/search-backend-node/src/extensions.ts | 31 +++++++ plugins/search-backend-node/src/index.ts | 8 +- plugins/search-backend-node/src/services.ts | 87 +++++++++++++++++++ plugins/search-backend-node/src/types.ts | 4 +- 6 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 plugins/search-backend-node/src/extensions.ts create mode 100644 plugins/search-backend-node/src/services.ts diff --git a/plugins/search-backend-node/package.json b/plugins/search-backend-node/package.json index 2abc0a8ef0..d77551ef29 100644 --- a/plugins/search-backend-node/package.json +++ b/plugins/search-backend-node/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts index 9397c098aa..907519f289 100644 --- a/plugins/search-backend-node/src/IndexBuilder.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { createServiceRef } from '@backstage/backend-plugin-api'; import { DocumentDecoratorFactory, DocumentTypeInfo, @@ -24,20 +23,16 @@ import { Transform, pipeline } from 'stream'; import { Logger } from 'winston'; import { Scheduler } from './Scheduler'; import { - IndexBuilderService, IndexBuilderOptions, RegisterCollatorParameters, RegisterDecoratorParameters, } from './types'; -export const searchIndexBuilderRef = createServiceRef({ - id: 'search.index-node', -}); /** * Used for adding collators, decorators and compile them into tasks which are added to a scheduler returned to the caller. * @public */ -export class IndexBuilder implements IndexBuilderService { +export class IndexBuilder { private collators: Record; private decorators: Record; private documentTypes: Record; diff --git a/plugins/search-backend-node/src/extensions.ts b/plugins/search-backend-node/src/extensions.ts new file mode 100644 index 0000000000..23392b70a5 --- /dev/null +++ b/plugins/search-backend-node/src/extensions.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createExtensionPoint } from '@backstage/backend-plugin-api'; +import { + SearchEngineRegistryExtensionPoint, + SearchIndexRegistryExtensionPoint, +} from './types'; + +export const searchEngineRegistryExtensionPoint = + createExtensionPoint({ + id: 'search.engine.registry', + }); + +export const searchIndexRegistryExtensionPoint = + createExtensionPoint({ + id: 'search.index.registry', + }); diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index 1bcd31de93..741410f7f8 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -20,7 +20,7 @@ * @packageDocumentation */ -export { IndexBuilder, searchIndexBuilderRef } from './IndexBuilder'; +export { IndexBuilder } from './IndexBuilder'; export { Scheduler } from './Scheduler'; export * from './collators'; export { LunrSearchEngine } from './engines'; @@ -33,9 +33,15 @@ export type { IndexBuilderOptions, RegisterCollatorParameters, RegisterDecoratorParameters, + SearchIndexRegistryExtensionPoint, + SearchEngineRegistryExtensionPoint, } from './types'; export * from './errors'; export * from './indexing'; export * from './test-utils'; export type { ScheduleTaskParameters } from './Scheduler'; + +// TODO: export as alfa subpath +export * from './services'; +export * from './extensions'; diff --git a/plugins/search-backend-node/src/services.ts b/plugins/search-backend-node/src/services.ts new file mode 100644 index 0000000000..7a39184cf8 --- /dev/null +++ b/plugins/search-backend-node/src/services.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; +import { + createServiceRef, + createServiceFactory, + coreServices, +} from '@backstage/backend-plugin-api'; +import { DocumentTypeInfo } from '@backstage/plugin-search-common'; +import { IndexBuilder } from './IndexBuilder'; +import { Scheduler } from './Scheduler'; +import { + IndexBuilderServiceBuildOptions, + SearchIndexBuilderService, +} from './types'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; + +type DefaultSearchIndexBuilderServiceOptions = { + logger: Logger; +}; + +class DefaultSearchIndexBuilderService implements SearchIndexBuilderService { + private logger: Logger; + private indexBuilder: IndexBuilder | null = null; + + private constructor(options: DefaultSearchIndexBuilderServiceOptions) { + this.logger = options.logger; + } + + static fromConfig(options: DefaultSearchIndexBuilderServiceOptions) { + return new DefaultSearchIndexBuilderService(options); + } + + build( + options: IndexBuilderServiceBuildOptions, + ): Promise<{ scheduler: Scheduler }> { + this.indexBuilder = new IndexBuilder({ + logger: this.logger, + searchEngine: options.searchEngine, + }); + + options.collators.forEach(collator => + this.indexBuilder?.addCollator(collator), + ); + + options.decorators.forEach(decorator => + this.indexBuilder?.addDecorator(decorator), + ); + + return this.indexBuilder?.build(); + } + + getDocumentTypes(): Record { + return this.indexBuilder?.getDocumentTypes() ?? {}; + } +} + +export const searchIndexBuilderService = + createServiceRef({ + id: 'search.index.builder', + defaultFactory: async service => + createServiceFactory({ + service, + deps: { + logger: coreServices.logger, + }, + factory({ logger }) { + return DefaultSearchIndexBuilderService.fromConfig({ + logger: loggerToWinstonLogger(logger), + }); + }, + }), + }); diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index 94b21cceec..4bf726bd0a 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -65,7 +65,7 @@ export type IndexBuilderServiceBuildOptions = { collators: RegisterCollatorParameters[]; decorators: RegisterDecoratorParameters[]; }; -export interface IndexBuilderService { +export interface SearchIndexBuilderService { build(options: IndexBuilderServiceBuildOptions): Promise<{ scheduler: Scheduler; }>; @@ -81,5 +81,5 @@ export interface SearchIndexRegistryExtensionPoint { export interface SearchEngineRegistryExtensionPoint { setSearchEngine(searchEngine: SearchEngine): void; - getSearchEngine(): SearchEngine; + getSearchEngine(): SearchEngine | null; } From 4f759aeef4ceb42c504ff0dabd1d1054654190d8 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 1 Mar 2023 21:15:20 +0100 Subject: [PATCH 03/27] feat(search): create search plugin implementation Signed-off-by: Camila Belo --- plugins/search-backend/package.json | 1 + plugins/search-backend/src/index.ts | 3 + plugins/search-backend/src/plugin.ts | 100 ++++++++++++++++++++++----- 3 files changed, 87 insertions(+), 17 deletions(-) diff --git a/plugins/search-backend/package.json b/plugins/search-backend/package.json index 02d4d53a92..bbd5d78f79 100644 --- a/plugins/search-backend/package.json +++ b/plugins/search-backend/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", diff --git a/plugins/search-backend/src/index.ts b/plugins/search-backend/src/index.ts index 851efd1875..0775a64378 100644 --- a/plugins/search-backend/src/index.ts +++ b/plugins/search-backend/src/index.ts @@ -21,3 +21,6 @@ */ export * from './service/router'; + +// TODO: export as alfa subpath +export { searchPlugin } from './plugin'; diff --git a/plugins/search-backend/src/plugin.ts b/plugins/search-backend/src/plugin.ts index 5d7c7935a8..5e17f03740 100644 --- a/plugins/search-backend/src/plugin.ts +++ b/plugins/search-backend/src/plugin.ts @@ -1,43 +1,109 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import { coreServices, createBackendPlugin, } from '@backstage/backend-plugin-api'; import { loggerToWinstonLogger } from '@backstage/backend-common'; import { - searchIndexBuilderRef, - searchEngineExtensionPoint, - searchIndexExtensionPoint, + searchIndexBuilderService, + searchIndexRegistryExtensionPoint, + SearchIndexRegistryExtensionPoint, + searchEngineRegistryExtensionPoint, + RegisterCollatorParameters, + RegisterDecoratorParameters, + SearchEngineRegistryExtensionPoint, + LunrSearchEngine, } from '@backstage/plugin-search-backend-node'; import { createRouter } from './service/router'; +import { SearchEngine } from '@backstage/plugin-search-common'; + +class SearchIndexRegistry implements SearchIndexRegistryExtensionPoint { + private collators: RegisterCollatorParameters[] = []; + private decorators: RegisterDecoratorParameters[] = []; + + public addCollator(options: RegisterCollatorParameters): void { + this.collators.push(options); + } + + public addDecorator(options: RegisterDecoratorParameters): void { + this.decorators.push(options); + } + + public getCollators(): RegisterCollatorParameters[] { + return this.collators; + } + + public getDecorators(): RegisterDecoratorParameters[] { + return this.decorators; + } +} + +class SearchEngineRegistry implements SearchEngineRegistryExtensionPoint { + private searchEngine: SearchEngine | null = null; + + public setSearchEngine(searchEngine: SearchEngine): void { + if (this.searchEngine) { + throw new Error('Multiple Search engines is not supported at this time'); + } + this.searchEngine = searchEngine; + } + + public getSearchEngine(): SearchEngine | null { + return this.searchEngine; + } +} export const searchPlugin = createBackendPlugin({ pluginId: 'search', register(env) { + const searchIndexRegistry = new SearchIndexRegistry(); + env.registerExtensionPoint( + searchIndexRegistryExtensionPoint, + searchIndexRegistry, + ); + + const searchEngineRegistry = new SearchEngineRegistry(); + env.registerExtensionPoint( + searchEngineRegistryExtensionPoint, + searchEngineRegistry, + ); + env.registerInit({ deps: { logger: coreServices.logger, config: coreServices.config, permissions: coreServices.permissions, http: coreServices.httpRouter, - searchIndexBuilder: searchIndexBuilderRef, - searchEngineRegistry: searchEngineExtensionPoint, - searchIndexRegistry: searchIndexExtensionPoint, + searchIndexBuilder: searchIndexBuilderService, }, - async init({ - config, - logger, - permissions, - http, - searchEngineRegistry, - searchIndexRegistry, - searchIndexBuilder, - }) { - const searchEngine = searchEngineRegistry.getSearchEngine(); + async init({ config, logger, permissions, http, searchIndexBuilder }) { + let searchEngine = searchEngineRegistry.getSearchEngine(); + if (!searchEngine) { + searchEngine = new LunrSearchEngine({ + logger: loggerToWinstonLogger(logger), + }); + } + const collators = searchIndexRegistry.getCollators(); const decorators = searchIndexRegistry.getDecorators(); - const { scheduler } = searchIndexBuilder.build({ + const { scheduler } = await searchIndexBuilder.build({ searchEngine, collators, decorators, From 406647ea6b8f73db3ae5db58b70acfff6a76322e Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 1 Mar 2023 21:15:54 +0100 Subject: [PATCH 04/27] feat(search): update yarn lock Signed-off-by: Camila Belo --- yarn.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yarn.lock b/yarn.lock index adc0ba595e..6615236acf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7812,6 +7812,7 @@ __metadata: resolution: "@backstage/plugin-search-backend-node@workspace:plugins/search-backend-node" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" @@ -7833,6 +7834,7 @@ __metadata: resolution: "@backstage/plugin-search-backend@workspace:plugins/search-backend" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/errors": "workspace:^" From 97e5d514d17350eb31f5cc9c7e282766da4a01e0 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Wed, 1 Mar 2023 21:47:19 +0100 Subject: [PATCH 05/27] create backend modules for search engines Signed-off-by: Emma Indal --- packages/backend-next/src/index.ts | 10 ++++++ .../src/engines/ElasticSearchSearchEngine.ts | 32 ++++++++++++++++++- .../src/engines/index.ts | 1 + .../src/index.ts | 1 + .../src/PgSearchEngine/PgSearchEngine.ts | 26 +++++++++++++++ .../src/PgSearchEngine/index.ts | 2 +- .../src/engines/LunrSearchEngine.ts | 24 ++++++++++++++ .../search-backend-node/src/engines/index.ts | 2 +- plugins/search-backend-node/src/index.ts | 2 +- plugins/search-backend/src/index.ts | 4 +-- 10 files changed, 97 insertions(+), 7 deletions(-) diff --git a/packages/backend-next/src/index.ts b/packages/backend-next/src/index.ts index 8f8984b192..ff7e5aa3db 100644 --- a/packages/backend-next/src/index.ts +++ b/packages/backend-next/src/index.ts @@ -20,6 +20,10 @@ import { createBackend } from '@backstage/backend-defaults'; import { appPlugin } from '@backstage/plugin-app-backend/alpha'; import { todoPlugin } from '@backstage/plugin-todo-backend'; import { techdocsPlugin } from '@backstage/plugin-techdocs-backend/alpha'; +import { searchPlugin } from '@backstage/plugin-search-backend'; +import { elasticSearchEngineModule } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { pgSearchEngineModule } from '@backstage/plugin-search-backend-module-pg'; +import { lunrSearchEngineModule } from '@backstage/plugin-search-backend-node/'; const backend = createBackend(); @@ -28,4 +32,10 @@ backend.add(catalogModuleTemplateKind()); backend.add(appPlugin({ appPackageName: 'example-app' })); backend.add(todoPlugin()); backend.add(techdocsPlugin()); + +backend.add(searchPlugin()); +backend.add(elasticSearchEngineModule()); +backend.add(pgSearchEngineModule()); +backend.add(lunrSearchEngineModule()); + backend.start(); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 25df3101de..bbeceabaa6 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -30,9 +30,17 @@ import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; import { ElasticSearchCustomIndexTemplate } from './types'; import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; import { Logger } from 'winston'; -import { MissingIndexError } from '@backstage/plugin-search-backend-node'; +import { + MissingIndexError, + searchEngineRegistryExtensionPoint, +} from '@backstage/plugin-search-backend-node'; import esb from 'elastic-builder'; import { v4 as uuid } from 'uuid'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; export type { ElasticSearchClientOptions }; @@ -521,3 +529,25 @@ export async function createElasticSearchClientOptions( : {}), }; } + +export const elasticSearchEngineModule = createBackendModule({ + moduleId: 'elasticSearchEngineModule', + pluginId: 'search-backend', + register(env) { + env.registerInit({ + deps: { + searchEngineRegistry: searchEngineRegistryExtensionPoint, + logger: coreServices.logger, + config: coreServices.config, + }, + async init({ searchEngineRegistry, logger, config }) { + searchEngineRegistry.setSearchEngine( + await ElasticSearchSearchEngine.fromConfig({ + logger: loggerToWinstonLogger(logger), + config: config, + }), + ); + }, + }); + }, +}); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts index cb24bdc56a..01f77034a0 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts @@ -17,6 +17,7 @@ export { decodePageCursor as decodeElasticSearchPageCursor, ElasticSearchSearchEngine, + elasticSearchEngineModule, } from './ElasticSearchSearchEngine'; export { isOpenSearchCompatible } from './ElasticSearchClientOptions'; export type { diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts index e0b05014cc..acff595246 100644 --- a/plugins/search-backend-module-elasticsearch/src/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/index.ts @@ -23,6 +23,7 @@ export { decodeElasticSearchPageCursor, ElasticSearchSearchEngine, + elasticSearchEngineModule, isOpenSearchCompatible, } from './engines'; export type { diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index d2cf80e0ca..7f1bb89754 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -30,6 +30,11 @@ import { import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node'; /** * Search query that the Postgres search engine understands. @@ -244,3 +249,24 @@ export function decodePageCursor(pageCursor?: string): { page: number } { export function encodePageCursor({ page }: { page: number }): string { return Buffer.from(`${page}`, 'utf-8').toString('base64'); } + +export const pgSearchEngineModule = createBackendModule({ + moduleId: 'pgSearchEngineModule', + pluginId: 'search-backend', + register(env) { + env.registerInit({ + deps: { + searchEngineRegistry: searchEngineRegistryExtensionPoint, + database: coreServices.database, + config: coreServices.config, + }, + async init({ searchEngineRegistry, database, config }) { + searchEngineRegistry.setSearchEngine( + await PgSearchEngine.fromConfig(config, { + database: database, + }), + ); + }, + }); + }, +}); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts index e62e709f26..7a90d9e3f7 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -export { PgSearchEngine } from './PgSearchEngine'; +export { PgSearchEngine, pgSearchEngineModule } from './PgSearchEngine'; export type { ConcretePgSearchQuery, PgSearchQueryTranslatorOptions, diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index b69b5ea863..0db9051e84 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -26,6 +26,12 @@ import lunr from 'lunr'; import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { searchEngineRegistryExtensionPoint } from '../extensions'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; /** * Type of translated query for the Lunr Search Engine. @@ -334,3 +340,21 @@ export function parseHighlightFields({ }), ); } + +export const lunrSearchEngineModule = createBackendModule({ + moduleId: 'lunrSearchEngineModule', + pluginId: 'search-backend', + register(env) { + env.registerInit({ + deps: { + searchEngineRegistry: searchEngineRegistryExtensionPoint, + logger: coreServices.logger, + }, + async init({ searchEngineRegistry, logger }) { + searchEngineRegistry.setSearchEngine( + new LunrSearchEngine({ logger: loggerToWinstonLogger(logger) }), + ); + }, + }); + }, +}); diff --git a/plugins/search-backend-node/src/engines/index.ts b/plugins/search-backend-node/src/engines/index.ts index 0d710eadc2..59aa4a24b0 100644 --- a/plugins/search-backend-node/src/engines/index.ts +++ b/plugins/search-backend-node/src/engines/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -export { LunrSearchEngine } from './LunrSearchEngine'; +export { LunrSearchEngine, lunrSearchEngineModule } from './LunrSearchEngine'; export type { ConcreteLunrQuery, LunrQueryTranslator, diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index 741410f7f8..7395036672 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -23,7 +23,7 @@ export { IndexBuilder } from './IndexBuilder'; export { Scheduler } from './Scheduler'; export * from './collators'; -export { LunrSearchEngine } from './engines'; +export { LunrSearchEngine, lunrSearchEngineModule } from './engines'; export type { ConcreteLunrQuery, LunrQueryTranslator, diff --git a/plugins/search-backend/src/index.ts b/plugins/search-backend/src/index.ts index 0775a64378..335841d383 100644 --- a/plugins/search-backend/src/index.ts +++ b/plugins/search-backend/src/index.ts @@ -21,6 +21,4 @@ */ export * from './service/router'; - -// TODO: export as alfa subpath -export { searchPlugin } from './plugin'; +export * from './plugin'; From cbf03b44646dbb03362ffc3d4222b810c03d72da Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Wed, 1 Mar 2023 22:13:22 +0100 Subject: [PATCH 06/27] add possibilities to pass options to elasticSearchEngineModule Signed-off-by: Emma Indal --- packages/backend-next/src/index.ts | 16 ++++-- .../src/engines/ElasticSearchSearchEngine.ts | 53 ++++++++++++------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/packages/backend-next/src/index.ts b/packages/backend-next/src/index.ts index ff7e5aa3db..071a446cae 100644 --- a/packages/backend-next/src/index.ts +++ b/packages/backend-next/src/index.ts @@ -22,8 +22,8 @@ import { todoPlugin } from '@backstage/plugin-todo-backend'; import { techdocsPlugin } from '@backstage/plugin-techdocs-backend/alpha'; import { searchPlugin } from '@backstage/plugin-search-backend'; import { elasticSearchEngineModule } from '@backstage/plugin-search-backend-module-elasticsearch'; -import { pgSearchEngineModule } from '@backstage/plugin-search-backend-module-pg'; -import { lunrSearchEngineModule } from '@backstage/plugin-search-backend-node/'; +// import { pgSearchEngineModule } from '@backstage/plugin-search-backend-module-pg'; +// import { lunrSearchEngineModule } from '@backstage/plugin-search-backend-node/'; const backend = createBackend(); @@ -34,8 +34,14 @@ backend.add(todoPlugin()); backend.add(techdocsPlugin()); backend.add(searchPlugin()); -backend.add(elasticSearchEngineModule()); -backend.add(pgSearchEngineModule()); -backend.add(lunrSearchEngineModule()); +// just as example of search engine module, remove before shipping and use default +backend.add( + elasticSearchEngineModule({ + indexTemplate: { + name: 'my-custom-template', + body: { index_patterns: ['*index*'], template: {} }, + }, + }), +); backend.start(); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index bbeceabaa6..288365a438 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -530,24 +530,39 @@ export async function createElasticSearchClientOptions( }; } -export const elasticSearchEngineModule = createBackendModule({ - moduleId: 'elasticSearchEngineModule', - pluginId: 'search-backend', - register(env) { - env.registerInit({ - deps: { - searchEngineRegistry: searchEngineRegistryExtensionPoint, - logger: coreServices.logger, - config: coreServices.config, - }, - async init({ searchEngineRegistry, logger, config }) { - searchEngineRegistry.setSearchEngine( - await ElasticSearchSearchEngine.fromConfig({ +export type ElasticSearchEngineModuleOptions = { + translator?: ElasticSearchQueryTranslator; + indexTemplate?: ElasticSearchCustomIndexTemplate; +}; + +export const elasticSearchEngineModule = createBackendModule( + (options?: ElasticSearchEngineModuleOptions) => ({ + moduleId: 'elasticSearchEngineModule', + pluginId: 'search-backend', + register(env) { + env.registerInit({ + deps: { + searchEngineRegistry: searchEngineRegistryExtensionPoint, + logger: coreServices.logger, + config: coreServices.config, + }, + async init({ searchEngineRegistry, logger, config }) { + const searchEngine = await ElasticSearchSearchEngine.fromConfig({ logger: loggerToWinstonLogger(logger), config: config, - }), - ); - }, - }); - }, -}); + }); + searchEngineRegistry.setSearchEngine(searchEngine); + // set custom translator if available + if (options?.translator) { + searchEngine.setTranslator(options.translator); + } + + // set custom index template if available TODO(emmaindal): make it possible to pass in mutliple index templates + if (options?.indexTemplate) { + searchEngine.setIndexTemplate(options.indexTemplate); + } + }, + }); + }, + }), +); From 58f758386151c718b034dba9d17297738e52d3e5 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 6 Mar 2023 08:29:03 +0100 Subject: [PATCH 07/27] create searchIndexRegistry module + add to backend Signed-off-by: Emma Indal --- packages/backend-next/src/index.ts | 48 ++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/packages/backend-next/src/index.ts b/packages/backend-next/src/index.ts index 071a446cae..93161de9a2 100644 --- a/packages/backend-next/src/index.ts +++ b/packages/backend-next/src/index.ts @@ -22,9 +22,56 @@ import { todoPlugin } from '@backstage/plugin-todo-backend'; import { techdocsPlugin } from '@backstage/plugin-techdocs-backend/alpha'; import { searchPlugin } from '@backstage/plugin-search-backend'; import { elasticSearchEngineModule } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; // import { pgSearchEngineModule } from '@backstage/plugin-search-backend-module-pg'; // import { lunrSearchEngineModule } from '@backstage/plugin-search-backend-node/'; +// TODO: move to search-backend-node? +export const searchIndexRegistry = createBackendModule({ + moduleId: 'searchIndexRegistry', + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + indexRegistry: searchIndexRegistryExtensionPoint, + config: coreServices.config, + discovery: coreServices.discovery, + tokenManager: coreServices.tokenManager, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + }, + async init({ + indexRegistry, + config, + discovery, + tokenManager, + scheduler, + }) { + const schedule = scheduler.createScheduledTaskRunner({ + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + // A 3 second delay gives the backend server a chance to initialize before + // any collators are executed, which may attempt requests against the API. + initialDelay: { seconds: 3 }, + }); + + indexRegistry.addCollator({ + schedule, + factory: DefaultCatalogCollatorFactory.fromConfig(config, { + discovery: discovery, + tokenManager: tokenManager, + }), + }); + }, + }); + }, +}); + const backend = createBackend(); backend.add(catalogPlugin()); @@ -43,5 +90,6 @@ backend.add( }, }), ); +backend.add(searchIndexRegistry()); backend.start(); From d5fe020807ee9e8cfe1c035cdff2dba5165bbbc1 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 6 Mar 2023 08:29:42 +0100 Subject: [PATCH 08/27] update plugin id to match naming conventions Signed-off-by: Emma Indal --- .../src/engines/ElasticSearchSearchEngine.ts | 2 +- .../src/PgSearchEngine/PgSearchEngine.ts | 2 +- plugins/search-backend-node/src/engines/LunrSearchEngine.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 288365a438..dbd8db7a0d 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -538,7 +538,7 @@ export type ElasticSearchEngineModuleOptions = { export const elasticSearchEngineModule = createBackendModule( (options?: ElasticSearchEngineModuleOptions) => ({ moduleId: 'elasticSearchEngineModule', - pluginId: 'search-backend', + pluginId: 'search', register(env) { env.registerInit({ deps: { diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 7f1bb89754..845c12fa3d 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -252,7 +252,7 @@ export function encodePageCursor({ page }: { page: number }): string { export const pgSearchEngineModule = createBackendModule({ moduleId: 'pgSearchEngineModule', - pluginId: 'search-backend', + pluginId: 'search', register(env) { env.registerInit({ deps: { diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index 0db9051e84..75875dc944 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -343,7 +343,7 @@ export function parseHighlightFields({ export const lunrSearchEngineModule = createBackendModule({ moduleId: 'lunrSearchEngineModule', - pluginId: 'search-backend', + pluginId: 'search', register(env) { env.registerInit({ deps: { From bf54977e6878cbf33bc8094c35a557cee5811bbe Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 6 Mar 2023 09:54:51 +0100 Subject: [PATCH 09/27] remove getter methods on interfaces Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- plugins/search-backend-node/src/types.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index 4bf726bd0a..e280e1a7ef 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -75,11 +75,8 @@ export interface SearchIndexBuilderService { export interface SearchIndexRegistryExtensionPoint { addCollator(options: RegisterCollatorParameters): void; addDecorator(options: RegisterDecoratorParameters): void; - getCollators(): RegisterCollatorParameters[]; - getDecorators(): RegisterDecoratorParameters[]; } export interface SearchEngineRegistryExtensionPoint { setSearchEngine(searchEngine: SearchEngine): void; - getSearchEngine(): SearchEngine | null; } From 1a79af33f2934442aacb0fd4f69c241f70152c71 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 6 Mar 2023 10:35:24 +0100 Subject: [PATCH 10/27] clean up comments and dependencies Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- packages/backend-next/package.json | 5 ++ packages/backend-next/src/index.ts | 60 +------------- packages/backend-next/src/plugins/search.ts | 82 +++++++++++++++++++ .../package.json | 2 + .../src/engines/ElasticSearchSearchEngine.ts | 2 +- plugins/search-backend/src/plugin.ts | 2 +- yarn.lock | 6 ++ 7 files changed, 98 insertions(+), 61 deletions(-) create mode 100644 packages/backend-next/src/plugins/search.ts diff --git a/packages/backend-next/package.json b/packages/backend-next/package.json index 14ec71f352..8df5ace282 100644 --- a/packages/backend-next/package.json +++ b/packages/backend-next/package.json @@ -25,10 +25,15 @@ "clean": "backstage-cli package clean" }, "dependencies": { + "@backstage/backend-common": "workspace:^", "@backstage/backend-defaults": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/plugin-app-backend": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-explore-backend": "workspace:^", "@backstage/plugin-scaffolder-backend": "workspace:^", + "@backstage/plugin-search-backend": "workspace:^", + "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-techdocs-backend": "workspace:^", "@backstage/plugin-todo-backend": "workspace:^" }, diff --git a/packages/backend-next/src/index.ts b/packages/backend-next/src/index.ts index 93161de9a2..cf275ae3ad 100644 --- a/packages/backend-next/src/index.ts +++ b/packages/backend-next/src/index.ts @@ -21,56 +21,7 @@ import { appPlugin } from '@backstage/plugin-app-backend/alpha'; import { todoPlugin } from '@backstage/plugin-todo-backend'; import { techdocsPlugin } from '@backstage/plugin-techdocs-backend/alpha'; import { searchPlugin } from '@backstage/plugin-search-backend'; -import { elasticSearchEngineModule } from '@backstage/plugin-search-backend-module-elasticsearch'; -import { - coreServices, - createBackendModule, -} from '@backstage/backend-plugin-api'; -import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node'; -import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; -// import { pgSearchEngineModule } from '@backstage/plugin-search-backend-module-pg'; -// import { lunrSearchEngineModule } from '@backstage/plugin-search-backend-node/'; - -// TODO: move to search-backend-node? -export const searchIndexRegistry = createBackendModule({ - moduleId: 'searchIndexRegistry', - pluginId: 'search', - register(env) { - env.registerInit({ - deps: { - indexRegistry: searchIndexRegistryExtensionPoint, - config: coreServices.config, - discovery: coreServices.discovery, - tokenManager: coreServices.tokenManager, - logger: coreServices.logger, - scheduler: coreServices.scheduler, - }, - async init({ - indexRegistry, - config, - discovery, - tokenManager, - scheduler, - }) { - const schedule = scheduler.createScheduledTaskRunner({ - frequency: { minutes: 10 }, - timeout: { minutes: 15 }, - // A 3 second delay gives the backend server a chance to initialize before - // any collators are executed, which may attempt requests against the API. - initialDelay: { seconds: 3 }, - }); - - indexRegistry.addCollator({ - schedule, - factory: DefaultCatalogCollatorFactory.fromConfig(config, { - discovery: discovery, - tokenManager: tokenManager, - }), - }); - }, - }); - }, -}); +import { searchIndexRegistry } from './plugins/search'; const backend = createBackend(); @@ -81,15 +32,6 @@ backend.add(todoPlugin()); backend.add(techdocsPlugin()); backend.add(searchPlugin()); -// just as example of search engine module, remove before shipping and use default -backend.add( - elasticSearchEngineModule({ - indexTemplate: { - name: 'my-custom-template', - body: { index_patterns: ['*index*'], template: {} }, - }, - }), -); backend.add(searchIndexRegistry()); backend.start(); diff --git a/packages/backend-next/src/plugins/search.ts b/packages/backend-next/src/plugins/search.ts new file mode 100644 index 0000000000..559e6a9cd1 --- /dev/null +++ b/packages/backend-next/src/plugins/search.ts @@ -0,0 +1,82 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; +import { ToolDocumentCollatorFactory } from '@backstage/plugin-explore-backend'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; + +export const searchIndexRegistry = createBackendModule({ + moduleId: 'searchIndexRegistry', + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + indexRegistry: searchIndexRegistryExtensionPoint, + config: coreServices.config, + discovery: coreServices.discovery, + tokenManager: coreServices.tokenManager, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + }, + async init({ + indexRegistry, + config, + logger, + discovery, + tokenManager, + scheduler, + }) { + const schedule = scheduler.createScheduledTaskRunner({ + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + // A 3 second delay gives the backend server a chance to initialize before + // any collators are executed, which may attempt requests against the API. + initialDelay: { seconds: 3 }, + }); + + indexRegistry.addCollator({ + schedule, + factory: DefaultCatalogCollatorFactory.fromConfig(config, { + discovery, + tokenManager, + }), + }); + + indexRegistry.addCollator({ + schedule, + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery, + logger: loggerToWinstonLogger(logger), + tokenManager, + }), + }); + + indexRegistry.addCollator({ + schedule, + factory: ToolDocumentCollatorFactory.fromConfig(config, { + discovery, + logger: loggerToWinstonLogger(logger), + }), + }); + }, + }); + }, +}); diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json index e854687360..aff861b784 100644 --- a/plugins/search-backend-module-elasticsearch/package.json +++ b/plugins/search-backend-module-elasticsearch/package.json @@ -23,6 +23,8 @@ "clean": "backstage-cli package clean" }, "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index dbd8db7a0d..f368799673 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -557,7 +557,7 @@ export const elasticSearchEngineModule = createBackendModule( searchEngine.setTranslator(options.translator); } - // set custom index template if available TODO(emmaindal): make it possible to pass in mutliple index templates + // set custom index template if available if (options?.indexTemplate) { searchEngine.setIndexTemplate(options.indexTemplate); } diff --git a/plugins/search-backend/src/plugin.ts b/plugins/search-backend/src/plugin.ts index 5e17f03740..35a9835ef6 100644 --- a/plugins/search-backend/src/plugin.ts +++ b/plugins/search-backend/src/plugin.ts @@ -117,7 +117,7 @@ export const searchPlugin = createBackendPlugin({ engine: searchEngine, types: searchIndexBuilder.getDocumentTypes(), }); - // We register the router with the http service. + http.use(router); }, }); diff --git a/yarn.lock b/yarn.lock index 6615236acf..3c787dceb3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7773,6 +7773,7 @@ __metadata: resolution: "@backstage/plugin-search-backend-module-elasticsearch@workspace:plugins/search-backend-module-elasticsearch" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" @@ -22481,11 +22482,16 @@ __metadata: version: 0.0.0-use.local resolution: "example-backend-next@workspace:packages/backend-next" dependencies: + "@backstage/backend-common": "workspace:^" "@backstage/backend-defaults": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/plugin-app-backend": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-explore-backend": "workspace:^" "@backstage/plugin-scaffolder-backend": "workspace:^" + "@backstage/plugin-search-backend": "workspace:^" + "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-techdocs-backend": "workspace:^" "@backstage/plugin-todo-backend": "workspace:^" languageName: unknown From 4cc252bdd5b5e404fe54383f1a220e24c93afbae Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 6 Mar 2023 11:22:17 +0100 Subject: [PATCH 11/27] use alpha exports Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- packages/backend-next/src/index.ts | 2 +- packages/backend-next/src/plugins/search.ts | 2 +- .../package.json | 19 +++++- .../src/alpha.ts | 63 +++++++++++++++++++ .../src/engines/ElasticSearchSearchEngine.ts | 47 +------------- .../src/engines/index.ts | 1 - .../src/index.ts | 1 - plugins/search-backend-module-pg/package.json | 20 +++++- .../src/PgSearchEngine/PgSearchEngine.ts | 26 -------- .../src/PgSearchEngine/index.ts | 2 +- plugins/search-backend-module-pg/src/alpha.ts | 42 +++++++++++++ plugins/search-backend-node/package.json | 19 +++++- .../src/{services.ts => alpha.ts} | 51 +++++++++++++-- .../src/engines/LunrSearchEngine.ts | 2 +- plugins/search-backend-node/src/index.ts | 6 -- plugins/search-backend-node/src/types.ts | 23 ------- plugins/search-backend/package.json | 19 +++++- .../src/alpha.ts} | 16 +---- plugins/search-backend/src/index.ts | 1 - plugins/search-backend/src/plugin.ts | 12 ++-- yarn.lock | 1 + 21 files changed, 229 insertions(+), 146 deletions(-) create mode 100644 plugins/search-backend-module-elasticsearch/src/alpha.ts create mode 100644 plugins/search-backend-module-pg/src/alpha.ts rename plugins/search-backend-node/src/{services.ts => alpha.ts} (67%) rename plugins/{search-backend-node/src/extensions.ts => search-backend/src/alpha.ts} (56%) diff --git a/packages/backend-next/src/index.ts b/packages/backend-next/src/index.ts index cf275ae3ad..10680ca798 100644 --- a/packages/backend-next/src/index.ts +++ b/packages/backend-next/src/index.ts @@ -20,7 +20,7 @@ import { createBackend } from '@backstage/backend-defaults'; import { appPlugin } from '@backstage/plugin-app-backend/alpha'; import { todoPlugin } from '@backstage/plugin-todo-backend'; import { techdocsPlugin } from '@backstage/plugin-techdocs-backend/alpha'; -import { searchPlugin } from '@backstage/plugin-search-backend'; +import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; import { searchIndexRegistry } from './plugins/search'; const backend = createBackend(); diff --git a/packages/backend-next/src/plugins/search.ts b/packages/backend-next/src/plugins/search.ts index 559e6a9cd1..1329135e56 100644 --- a/packages/backend-next/src/plugins/search.ts +++ b/packages/backend-next/src/plugins/search.ts @@ -17,7 +17,7 @@ import { coreServices, createBackendModule, } from '@backstage/backend-plugin-api'; -import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; import { ToolDocumentCollatorFactory } from '@backstage/plugin-explore-backend'; diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json index aff861b784..3c16bd1a7d 100644 --- a/plugins/search-backend-module-elasticsearch/package.json +++ b/plugins/search-backend-module-elasticsearch/package.json @@ -6,9 +6,22 @@ "types": "src/index.ts", "license": "Apache-2.0", "publishConfig": { - "access": "public", - "main": "dist/index.cjs.js", - "types": "dist/index.d.ts" + "access": "public" + }, + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } }, "backstage": { "role": "backend-plugin-module" diff --git a/plugins/search-backend-module-elasticsearch/src/alpha.ts b/plugins/search-backend-module-elasticsearch/src/alpha.ts new file mode 100644 index 0000000000..ad83919ee4 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/alpha.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + ElasticSearchCustomIndexTemplate, + ElasticSearchQueryTranslator, + ElasticSearchSearchEngine, +} from './engines'; + +export type ElasticSearchEngineModuleOptions = { + translator?: ElasticSearchQueryTranslator; + indexTemplate?: ElasticSearchCustomIndexTemplate; +}; + +export const elasticSearchEngineModule = createBackendModule( + (options?: ElasticSearchEngineModuleOptions) => ({ + moduleId: 'elasticSearchEngineModule', + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + searchEngineRegistry: searchEngineRegistryExtensionPoint, + logger: coreServices.logger, + config: coreServices.config, + }, + async init({ searchEngineRegistry, logger, config }) { + const searchEngine = await ElasticSearchSearchEngine.fromConfig({ + logger: loggerToWinstonLogger(logger), + config: config, + }); + searchEngineRegistry.setSearchEngine(searchEngine); + // set custom translator if available + if (options?.translator) { + searchEngine.setTranslator(options.translator); + } + + // set custom index template if available + if (options?.indexTemplate) { + searchEngine.setIndexTemplate(options.indexTemplate); + } + }, + }); + }, + }), +); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index f368799673..25df3101de 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -30,17 +30,9 @@ import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; import { ElasticSearchCustomIndexTemplate } from './types'; import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; import { Logger } from 'winston'; -import { - MissingIndexError, - searchEngineRegistryExtensionPoint, -} from '@backstage/plugin-search-backend-node'; +import { MissingIndexError } from '@backstage/plugin-search-backend-node'; import esb from 'elastic-builder'; import { v4 as uuid } from 'uuid'; -import { - coreServices, - createBackendModule, -} from '@backstage/backend-plugin-api'; -import { loggerToWinstonLogger } from '@backstage/backend-common'; export type { ElasticSearchClientOptions }; @@ -529,40 +521,3 @@ export async function createElasticSearchClientOptions( : {}), }; } - -export type ElasticSearchEngineModuleOptions = { - translator?: ElasticSearchQueryTranslator; - indexTemplate?: ElasticSearchCustomIndexTemplate; -}; - -export const elasticSearchEngineModule = createBackendModule( - (options?: ElasticSearchEngineModuleOptions) => ({ - moduleId: 'elasticSearchEngineModule', - pluginId: 'search', - register(env) { - env.registerInit({ - deps: { - searchEngineRegistry: searchEngineRegistryExtensionPoint, - logger: coreServices.logger, - config: coreServices.config, - }, - async init({ searchEngineRegistry, logger, config }) { - const searchEngine = await ElasticSearchSearchEngine.fromConfig({ - logger: loggerToWinstonLogger(logger), - config: config, - }); - searchEngineRegistry.setSearchEngine(searchEngine); - // set custom translator if available - if (options?.translator) { - searchEngine.setTranslator(options.translator); - } - - // set custom index template if available - if (options?.indexTemplate) { - searchEngine.setIndexTemplate(options.indexTemplate); - } - }, - }); - }, - }), -); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts index 01f77034a0..cb24bdc56a 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts @@ -17,7 +17,6 @@ export { decodePageCursor as decodeElasticSearchPageCursor, ElasticSearchSearchEngine, - elasticSearchEngineModule, } from './ElasticSearchSearchEngine'; export { isOpenSearchCompatible } from './ElasticSearchClientOptions'; export type { diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts index acff595246..e0b05014cc 100644 --- a/plugins/search-backend-module-elasticsearch/src/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/index.ts @@ -23,7 +23,6 @@ export { decodeElasticSearchPageCursor, ElasticSearchSearchEngine, - elasticSearchEngineModule, isOpenSearchCompatible, } from './engines'; export type { diff --git a/plugins/search-backend-module-pg/package.json b/plugins/search-backend-module-pg/package.json index b284557369..be1762394d 100644 --- a/plugins/search-backend-module-pg/package.json +++ b/plugins/search-backend-module-pg/package.json @@ -6,9 +6,22 @@ "types": "src/index.ts", "license": "Apache-2.0", "publishConfig": { - "access": "public", - "main": "dist/index.cjs.js", - "types": "dist/index.d.ts" + "access": "public" + }, + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } }, "backstage": { "role": "backend-plugin-module" @@ -24,6 +37,7 @@ }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 845c12fa3d..d2cf80e0ca 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -30,11 +30,6 @@ import { import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { - coreServices, - createBackendModule, -} from '@backstage/backend-plugin-api'; -import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node'; /** * Search query that the Postgres search engine understands. @@ -249,24 +244,3 @@ export function decodePageCursor(pageCursor?: string): { page: number } { export function encodePageCursor({ page }: { page: number }): string { return Buffer.from(`${page}`, 'utf-8').toString('base64'); } - -export const pgSearchEngineModule = createBackendModule({ - moduleId: 'pgSearchEngineModule', - pluginId: 'search', - register(env) { - env.registerInit({ - deps: { - searchEngineRegistry: searchEngineRegistryExtensionPoint, - database: coreServices.database, - config: coreServices.config, - }, - async init({ searchEngineRegistry, database, config }) { - searchEngineRegistry.setSearchEngine( - await PgSearchEngine.fromConfig(config, { - database: database, - }), - ); - }, - }); - }, -}); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts index 7a90d9e3f7..e62e709f26 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -export { PgSearchEngine, pgSearchEngineModule } from './PgSearchEngine'; +export { PgSearchEngine } from './PgSearchEngine'; export type { ConcretePgSearchQuery, PgSearchQueryTranslatorOptions, diff --git a/plugins/search-backend-module-pg/src/alpha.ts b/plugins/search-backend-module-pg/src/alpha.ts new file mode 100644 index 0000000000..bd12b4f118 --- /dev/null +++ b/plugins/search-backend-module-pg/src/alpha.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; +import { PgSearchEngine } from './PgSearchEngine'; + +export const pgSearchEngineModule = createBackendModule({ + moduleId: 'pgSearchEngineModule', + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + searchEngineRegistry: searchEngineRegistryExtensionPoint, + database: coreServices.database, + config: coreServices.config, + }, + async init({ searchEngineRegistry, database, config }) { + searchEngineRegistry.setSearchEngine( + await PgSearchEngine.fromConfig(config, { + database: database, + }), + ); + }, + }); + }, +}); diff --git a/plugins/search-backend-node/package.json b/plugins/search-backend-node/package.json index d77551ef29..869f008de1 100644 --- a/plugins/search-backend-node/package.json +++ b/plugins/search-backend-node/package.json @@ -6,9 +6,22 @@ "types": "src/index.ts", "license": "Apache-2.0", "publishConfig": { - "access": "public", - "main": "dist/index.cjs.js", - "types": "dist/index.d.ts" + "access": "public" + }, + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } }, "backstage": { "role": "node-library" diff --git a/plugins/search-backend-node/src/services.ts b/plugins/search-backend-node/src/alpha.ts similarity index 67% rename from plugins/search-backend-node/src/services.ts rename to plugins/search-backend-node/src/alpha.ts index 7a39184cf8..b2ca6570e1 100644 --- a/plugins/search-backend-node/src/services.ts +++ b/plugins/search-backend-node/src/alpha.ts @@ -15,19 +15,42 @@ */ import { Logger } from 'winston'; + import { createServiceRef, createServiceFactory, coreServices, } from '@backstage/backend-plugin-api'; -import { DocumentTypeInfo } from '@backstage/plugin-search-common'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + DocumentTypeInfo, + SearchEngine, +} from '@backstage/plugin-search-common'; +import { createExtensionPoint } from '@backstage/backend-plugin-api'; + +import { + RegisterCollatorParameters, + RegisterDecoratorParameters, +} from './types'; + import { IndexBuilder } from './IndexBuilder'; import { Scheduler } from './Scheduler'; -import { - IndexBuilderServiceBuildOptions, - SearchIndexBuilderService, -} from './types'; -import { loggerToWinstonLogger } from '@backstage/backend-common'; + +export interface SearchIndexBuilderService { + build(options: IndexBuilderServiceBuildOptions): Promise<{ + scheduler: Scheduler; + }>; + getDocumentTypes(): Record; +} + +export interface SearchIndexRegistryExtensionPoint { + addCollator(options: RegisterCollatorParameters): void; + addDecorator(options: RegisterDecoratorParameters): void; +} + +export interface SearchEngineRegistryExtensionPoint { + setSearchEngine(searchEngine: SearchEngine): void; +} type DefaultSearchIndexBuilderServiceOptions = { logger: Logger; @@ -85,3 +108,19 @@ export const searchIndexBuilderService = }, }), }); + +export const searchEngineRegistryExtensionPoint = + createExtensionPoint({ + id: 'search.engine.registry', + }); + +export const searchIndexRegistryExtensionPoint = + createExtensionPoint({ + id: 'search.index.registry', + }); + +export type IndexBuilderServiceBuildOptions = { + searchEngine: SearchEngine; + collators: RegisterCollatorParameters[]; + decorators: RegisterDecoratorParameters[]; +}; diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index 75875dc944..57fe123ede 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -30,7 +30,7 @@ import { coreServices, createBackendModule, } from '@backstage/backend-plugin-api'; -import { searchEngineRegistryExtensionPoint } from '../extensions'; +import { searchEngineRegistryExtensionPoint } from '../alpha'; import { loggerToWinstonLogger } from '@backstage/backend-common'; /** diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index 7395036672..140c0a22f8 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -33,15 +33,9 @@ export type { IndexBuilderOptions, RegisterCollatorParameters, RegisterDecoratorParameters, - SearchIndexRegistryExtensionPoint, - SearchEngineRegistryExtensionPoint, } from './types'; export * from './errors'; export * from './indexing'; export * from './test-utils'; export type { ScheduleTaskParameters } from './Scheduler'; - -// TODO: export as alfa subpath -export * from './services'; -export * from './extensions'; diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index e280e1a7ef..dfcf4d12ce 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -18,11 +18,9 @@ import { TaskRunner } from '@backstage/backend-tasks'; import { DocumentCollatorFactory, DocumentDecoratorFactory, - DocumentTypeInfo, SearchEngine, } from '@backstage/plugin-search-common'; import { Logger } from 'winston'; -import { Scheduler } from './Scheduler'; /** * Options required to instantiate the index builder. @@ -59,24 +57,3 @@ export interface RegisterDecoratorParameters { */ factory: DocumentDecoratorFactory; } - -export type IndexBuilderServiceBuildOptions = { - searchEngine: SearchEngine; - collators: RegisterCollatorParameters[]; - decorators: RegisterDecoratorParameters[]; -}; -export interface SearchIndexBuilderService { - build(options: IndexBuilderServiceBuildOptions): Promise<{ - scheduler: Scheduler; - }>; - getDocumentTypes(): Record; -} - -export interface SearchIndexRegistryExtensionPoint { - addCollator(options: RegisterCollatorParameters): void; - addDecorator(options: RegisterDecoratorParameters): void; -} - -export interface SearchEngineRegistryExtensionPoint { - setSearchEngine(searchEngine: SearchEngine): void; -} diff --git a/plugins/search-backend/package.json b/plugins/search-backend/package.json index bbd5d78f79..6e971ba646 100644 --- a/plugins/search-backend/package.json +++ b/plugins/search-backend/package.json @@ -6,9 +6,22 @@ "types": "src/index.ts", "license": "Apache-2.0", "publishConfig": { - "access": "public", - "main": "dist/index.cjs.js", - "types": "dist/index.d.ts" + "access": "public" + }, + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } }, "backstage": { "role": "backend-plugin" diff --git a/plugins/search-backend-node/src/extensions.ts b/plugins/search-backend/src/alpha.ts similarity index 56% rename from plugins/search-backend-node/src/extensions.ts rename to plugins/search-backend/src/alpha.ts index 23392b70a5..e35b33188b 100644 --- a/plugins/search-backend-node/src/extensions.ts +++ b/plugins/search-backend/src/alpha.ts @@ -14,18 +14,4 @@ * limitations under the License. */ -import { createExtensionPoint } from '@backstage/backend-plugin-api'; -import { - SearchEngineRegistryExtensionPoint, - SearchIndexRegistryExtensionPoint, -} from './types'; - -export const searchEngineRegistryExtensionPoint = - createExtensionPoint({ - id: 'search.engine.registry', - }); - -export const searchIndexRegistryExtensionPoint = - createExtensionPoint({ - id: 'search.index.registry', - }); +export { searchPlugin } from './plugin'; diff --git a/plugins/search-backend/src/index.ts b/plugins/search-backend/src/index.ts index 335841d383..851efd1875 100644 --- a/plugins/search-backend/src/index.ts +++ b/plugins/search-backend/src/index.ts @@ -21,4 +21,3 @@ */ export * from './service/router'; -export * from './plugin'; diff --git a/plugins/search-backend/src/plugin.ts b/plugins/search-backend/src/plugin.ts index 35a9835ef6..7c01ddc607 100644 --- a/plugins/search-backend/src/plugin.ts +++ b/plugins/search-backend/src/plugin.ts @@ -19,16 +19,18 @@ import { createBackendPlugin, } from '@backstage/backend-plugin-api'; import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + RegisterCollatorParameters, + RegisterDecoratorParameters, + LunrSearchEngine, +} from '@backstage/plugin-search-backend-node'; import { searchIndexBuilderService, searchIndexRegistryExtensionPoint, SearchIndexRegistryExtensionPoint, - searchEngineRegistryExtensionPoint, - RegisterCollatorParameters, - RegisterDecoratorParameters, SearchEngineRegistryExtensionPoint, - LunrSearchEngine, -} from '@backstage/plugin-search-backend-node'; + searchEngineRegistryExtensionPoint, +} from '@backstage/plugin-search-backend-node/alpha'; import { createRouter } from './service/router'; import { SearchEngine } from '@backstage/plugin-search-common'; diff --git a/yarn.lock b/yarn.lock index 3c787dceb3..c2c60036fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7796,6 +7796,7 @@ __metadata: resolution: "@backstage/plugin-search-backend-module-pg@workspace:plugins/search-backend-module-pg" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" From 5db02fc935a7baaf1dfecfb8eadce776052ce7e3 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 6 Mar 2023 15:00:11 +0100 Subject: [PATCH 12/27] generate api reports Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- .../alpha-api-report.md | 61 +++++++++ .../src/alpha.ts | 29 +++- .../alpha-api-report.md | 12 ++ plugins/search-backend-module-pg/src/alpha.ts | 4 + .../search-backend-node/alpha-api-report.md | 62 +++++++++ plugins/search-backend-node/src/alpha.ts | 111 ++++++++++----- .../src/engines/LunrSearchEngine.ts | 24 ---- .../search-backend-node/src/engines/index.ts | 2 +- plugins/search-backend-node/src/index.ts | 2 +- plugins/search-backend/alpha-api-report.md | 12 ++ plugins/search-backend/src/alpha.ts | 115 +++++++++++++++- plugins/search-backend/src/plugin.ts | 127 ------------------ 12 files changed, 369 insertions(+), 192 deletions(-) create mode 100644 plugins/search-backend-module-elasticsearch/alpha-api-report.md create mode 100644 plugins/search-backend-module-pg/alpha-api-report.md create mode 100644 plugins/search-backend-node/alpha-api-report.md create mode 100644 plugins/search-backend/alpha-api-report.md delete mode 100644 plugins/search-backend/src/plugin.ts diff --git a/plugins/search-backend-module-elasticsearch/alpha-api-report.md b/plugins/search-backend-module-elasticsearch/alpha-api-report.md new file mode 100644 index 0000000000..acdd14e781 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/alpha-api-report.md @@ -0,0 +1,61 @@ +## API Report File for "@backstage/plugin-search-backend-module-elasticsearch" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { SearchQuery } from '@backstage/plugin-search-common'; + +// @public +export type ElasticSearchConcreteQuery = { + documentTypes?: string[]; + elasticSearchQuery: Object; + pageSize: number; +}; + +// @public +export type ElasticSearchCustomIndexTemplate = { + name: string; + body: ElasticSearchCustomIndexTemplateBody; +}; + +// @public +export type ElasticSearchCustomIndexTemplateBody = { + index_patterns: string[]; + composed_of?: string[]; + template?: Record; +}; + +// @alpha +export const elasticSearchEngineModule: ( + options?: ElasticsearchEngineModuleOptions | undefined, +) => BackendFeature; + +// @alpha +export type ElasticsearchEngineModuleOptions = { + translator?: ElasticSearchQueryTranslator; + indexTemplate?: ElasticSearchCustomIndexTemplate; +}; + +// @public (undocumented) +export type ElasticSearchHighlightConfig = { + fragmentDelimiter: string; + fragmentSize: number; + numFragments: number; + preTag: string; + postTag: string; +}; + +// @public +export type ElasticSearchQueryTranslator = ( + query: SearchQuery, + options?: ElasticSearchQueryTranslatorOptions, +) => ElasticSearchConcreteQuery; + +// @public +export type ElasticSearchQueryTranslatorOptions = { + highlightOptions?: ElasticSearchHighlightConfig; +}; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/search-backend-module-elasticsearch/src/alpha.ts b/plugins/search-backend-module-elasticsearch/src/alpha.ts index ad83919ee4..8e75c9f603 100644 --- a/plugins/search-backend-module-elasticsearch/src/alpha.ts +++ b/plugins/search-backend-module-elasticsearch/src/alpha.ts @@ -20,18 +20,30 @@ import { } from '@backstage/backend-plugin-api'; import { loggerToWinstonLogger } from '@backstage/backend-common'; import { + ElasticSearchHighlightConfig, + ElasticSearchQueryTranslatorOptions, + ElasticSearchConcreteQuery, ElasticSearchCustomIndexTemplate, + ElasticSearchCustomIndexTemplateBody, ElasticSearchQueryTranslator, ElasticSearchSearchEngine, } from './engines'; -export type ElasticSearchEngineModuleOptions = { +/** + * @alpha + * Options for {@link elasticSearchEngineModule}. + */ +export type ElasticsearchEngineModuleOptions = { translator?: ElasticSearchQueryTranslator; indexTemplate?: ElasticSearchCustomIndexTemplate; }; +/** + * @alpha + * Search backend module for the Elasticsearch engine. + */ export const elasticSearchEngineModule = createBackendModule( - (options?: ElasticSearchEngineModuleOptions) => ({ + (options?: ElasticsearchEngineModuleOptions) => ({ moduleId: 'elasticSearchEngineModule', pluginId: 'search', register(env) { @@ -46,7 +58,7 @@ export const elasticSearchEngineModule = createBackendModule( logger: loggerToWinstonLogger(logger), config: config, }); - searchEngineRegistry.setSearchEngine(searchEngine); + // set custom translator if available if (options?.translator) { searchEngine.setTranslator(options.translator); @@ -56,8 +68,19 @@ export const elasticSearchEngineModule = createBackendModule( if (options?.indexTemplate) { searchEngine.setIndexTemplate(options.indexTemplate); } + + searchEngineRegistry.setSearchEngine(searchEngine); }, }); }, }), ); + +export type { + ElasticSearchCustomIndexTemplate, + ElasticSearchCustomIndexTemplateBody, + ElasticSearchQueryTranslator, + ElasticSearchQueryTranslatorOptions, + ElasticSearchHighlightConfig, + ElasticSearchConcreteQuery, +}; diff --git a/plugins/search-backend-module-pg/alpha-api-report.md b/plugins/search-backend-module-pg/alpha-api-report.md new file mode 100644 index 0000000000..01a83bace4 --- /dev/null +++ b/plugins/search-backend-module-pg/alpha-api-report.md @@ -0,0 +1,12 @@ +## API Report File for "@backstage/plugin-search-backend-module-pg" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; + +// @alpha +export const pgSearchEngineModule: () => BackendFeature; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/search-backend-module-pg/src/alpha.ts b/plugins/search-backend-module-pg/src/alpha.ts index bd12b4f118..80ce31adf6 100644 --- a/plugins/search-backend-module-pg/src/alpha.ts +++ b/plugins/search-backend-module-pg/src/alpha.ts @@ -20,6 +20,10 @@ import { import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; import { PgSearchEngine } from './PgSearchEngine'; +/** + * @alpha + * Search backend module for the Postgres engine. + */ export const pgSearchEngineModule = createBackendModule({ moduleId: 'pgSearchEngineModule', pluginId: 'search', diff --git a/plugins/search-backend-node/alpha-api-report.md b/plugins/search-backend-node/alpha-api-report.md new file mode 100644 index 0000000000..f53f86b709 --- /dev/null +++ b/plugins/search-backend-node/alpha-api-report.md @@ -0,0 +1,62 @@ +## API Report File for "@backstage/plugin-search-backend-node" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { DocumentDecoratorFactory } from '@backstage/plugin-search-common'; +import { DocumentTypeInfo } from '@backstage/plugin-search-common'; +import { ExtensionPoint } from '@backstage/backend-plugin-api'; +import { SearchEngine } from '@backstage/plugin-search-common'; +import { ServiceRef } from '@backstage/backend-plugin-api'; +import { TaskRunner } from '@backstage/backend-tasks'; + +// @alpha +export type IndexServiceBuildOptions = { + searchEngine: SearchEngine; + collators: RegisterCollatorParameters[]; + decorators: RegisterDecoratorParameters[]; +}; + +// @public +export interface RegisterCollatorParameters { + factory: DocumentCollatorFactory; + schedule: TaskRunner; +} + +// @public +export interface RegisterDecoratorParameters { + factory: DocumentDecoratorFactory; +} + +// @alpha +export interface SearchEngineRegistryExtensionPoint { + // (undocumented) + setSearchEngine(searchEngine: SearchEngine): void; +} + +// @alpha +export const searchEngineRegistryExtensionPoint: ExtensionPoint; + +// @alpha +export interface SearchIndexRegistryExtensionPoint { + // (undocumented) + addCollator(options: RegisterCollatorParameters): void; + // (undocumented) + addDecorator(options: RegisterDecoratorParameters): void; +} + +// @alpha +export const searchIndexRegistryExtensionPoint: ExtensionPoint; + +// @alpha +export interface SearchIndexService { + getDocumentTypes(): Record; + start(options: IndexServiceBuildOptions): Promise; +} + +// @alpha +export const searchIndexService: ServiceRef; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/search-backend-node/src/alpha.ts b/plugins/search-backend-node/src/alpha.ts index b2ca6570e1..22c6508b15 100644 --- a/plugins/search-backend-node/src/alpha.ts +++ b/plugins/search-backend-node/src/alpha.ts @@ -34,43 +34,70 @@ import { } from './types'; import { IndexBuilder } from './IndexBuilder'; -import { Scheduler } from './Scheduler'; -export interface SearchIndexBuilderService { - build(options: IndexBuilderServiceBuildOptions): Promise<{ - scheduler: Scheduler; - }>; +/** + * @alpha + * Options for build method on {@link SearchIndexService}. + */ +export type IndexServiceBuildOptions = { + searchEngine: SearchEngine; + collators: RegisterCollatorParameters[]; + decorators: RegisterDecoratorParameters[]; +}; + +/** + * @alpha + * Interface for implementation of index service. + */ +export interface SearchIndexService { + /** + * Starts indexing process + */ + start(options: IndexServiceBuildOptions): Promise; + /** + * Returns an index types list. + */ getDocumentTypes(): Record; } +/** + * @alpha + * Interface for search index registry extension point. + */ export interface SearchIndexRegistryExtensionPoint { addCollator(options: RegisterCollatorParameters): void; addDecorator(options: RegisterDecoratorParameters): void; } +/** + * @alpha + * Interface for search engine registry extension point. + */ export interface SearchEngineRegistryExtensionPoint { setSearchEngine(searchEngine: SearchEngine): void; } -type DefaultSearchIndexBuilderServiceOptions = { +type DefaultSearchIndexServiceOptions = { logger: Logger; }; -class DefaultSearchIndexBuilderService implements SearchIndexBuilderService { +/** + * @alpha + * Reponsible for register the indexing task and start the schedule. + */ +class DefaultSearchIndexService implements SearchIndexService { private logger: Logger; private indexBuilder: IndexBuilder | null = null; - private constructor(options: DefaultSearchIndexBuilderServiceOptions) { + private constructor(options: DefaultSearchIndexServiceOptions) { this.logger = options.logger; } - static fromConfig(options: DefaultSearchIndexBuilderServiceOptions) { - return new DefaultSearchIndexBuilderService(options); + static fromConfig(options: DefaultSearchIndexServiceOptions) { + return new DefaultSearchIndexService(options); } - build( - options: IndexBuilderServiceBuildOptions, - ): Promise<{ scheduler: Scheduler }> { + async start(options: IndexServiceBuildOptions): Promise { this.indexBuilder = new IndexBuilder({ logger: this.logger, searchEngine: options.searchEngine, @@ -84,7 +111,8 @@ class DefaultSearchIndexBuilderService implements SearchIndexBuilderService { this.indexBuilder?.addDecorator(decorator), ); - return this.indexBuilder?.build(); + const { scheduler } = await this.indexBuilder?.build(); + scheduler.start(); } getDocumentTypes(): Record { @@ -92,35 +120,48 @@ class DefaultSearchIndexBuilderService implements SearchIndexBuilderService { } } -export const searchIndexBuilderService = - createServiceRef({ - id: 'search.index.builder', - defaultFactory: async service => - createServiceFactory({ - service, - deps: { - logger: coreServices.logger, - }, - factory({ logger }) { - return DefaultSearchIndexBuilderService.fromConfig({ - logger: loggerToWinstonLogger(logger), - }); - }, - }), - }); +/** + * @alpha + * Service that builds a search index. + */ +export const searchIndexServiceRef = createServiceRef({ + id: 'search.index.service', + defaultFactory: async service => + createServiceFactory({ + service, + deps: { + logger: coreServices.logger, + }, + factory({ logger }) { + return DefaultSearchIndexService.fromConfig({ + logger: loggerToWinstonLogger(logger), + }); + }, + }), +}); +/** + * @alpha + * Extension point for register a search engine. + */ export const searchEngineRegistryExtensionPoint = createExtensionPoint({ id: 'search.engine.registry', }); +/** + * @alpha + * Extension point for registering collators and decorators + */ export const searchIndexRegistryExtensionPoint = createExtensionPoint({ id: 'search.index.registry', }); -export type IndexBuilderServiceBuildOptions = { - searchEngine: SearchEngine; - collators: RegisterCollatorParameters[]; - decorators: RegisterDecoratorParameters[]; -}; +/** + * @alpha + */ +export type { + RegisterCollatorParameters, + RegisterDecoratorParameters, +} from './types'; diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index 57fe123ede..b69b5ea863 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -26,12 +26,6 @@ import lunr from 'lunr'; import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; -import { - coreServices, - createBackendModule, -} from '@backstage/backend-plugin-api'; -import { searchEngineRegistryExtensionPoint } from '../alpha'; -import { loggerToWinstonLogger } from '@backstage/backend-common'; /** * Type of translated query for the Lunr Search Engine. @@ -340,21 +334,3 @@ export function parseHighlightFields({ }), ); } - -export const lunrSearchEngineModule = createBackendModule({ - moduleId: 'lunrSearchEngineModule', - pluginId: 'search', - register(env) { - env.registerInit({ - deps: { - searchEngineRegistry: searchEngineRegistryExtensionPoint, - logger: coreServices.logger, - }, - async init({ searchEngineRegistry, logger }) { - searchEngineRegistry.setSearchEngine( - new LunrSearchEngine({ logger: loggerToWinstonLogger(logger) }), - ); - }, - }); - }, -}); diff --git a/plugins/search-backend-node/src/engines/index.ts b/plugins/search-backend-node/src/engines/index.ts index 59aa4a24b0..0d710eadc2 100644 --- a/plugins/search-backend-node/src/engines/index.ts +++ b/plugins/search-backend-node/src/engines/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -export { LunrSearchEngine, lunrSearchEngineModule } from './LunrSearchEngine'; +export { LunrSearchEngine } from './LunrSearchEngine'; export type { ConcreteLunrQuery, LunrQueryTranslator, diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index 140c0a22f8..a188509c1c 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -23,7 +23,7 @@ export { IndexBuilder } from './IndexBuilder'; export { Scheduler } from './Scheduler'; export * from './collators'; -export { LunrSearchEngine, lunrSearchEngineModule } from './engines'; +export { LunrSearchEngine } from './engines'; export type { ConcreteLunrQuery, LunrQueryTranslator, diff --git a/plugins/search-backend/alpha-api-report.md b/plugins/search-backend/alpha-api-report.md new file mode 100644 index 0000000000..6ee94a0e29 --- /dev/null +++ b/plugins/search-backend/alpha-api-report.md @@ -0,0 +1,12 @@ +## API Report File for "@backstage/plugin-search-backend" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; + +// @alpha +export const searchPlugin: () => BackendFeature; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/search-backend/src/alpha.ts b/plugins/search-backend/src/alpha.ts index e35b33188b..0c14852875 100644 --- a/plugins/search-backend/src/alpha.ts +++ b/plugins/search-backend/src/alpha.ts @@ -14,4 +14,117 @@ * limitations under the License. */ -export { searchPlugin } from './plugin'; +import { + coreServices, + createBackendPlugin, +} from '@backstage/backend-plugin-api'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + RegisterCollatorParameters, + RegisterDecoratorParameters, + LunrSearchEngine, +} from '@backstage/plugin-search-backend-node'; +import { + searchIndexServiceRef, + searchIndexRegistryExtensionPoint, + SearchIndexRegistryExtensionPoint, + SearchEngineRegistryExtensionPoint, + searchEngineRegistryExtensionPoint, +} from '@backstage/plugin-search-backend-node/alpha'; + +import { createRouter } from './service/router'; +import { SearchEngine } from '@backstage/plugin-search-common'; + +class SearchIndexRegistry implements SearchIndexRegistryExtensionPoint { + private collators: RegisterCollatorParameters[] = []; + private decorators: RegisterDecoratorParameters[] = []; + + public addCollator(options: RegisterCollatorParameters): void { + this.collators.push(options); + } + + public addDecorator(options: RegisterDecoratorParameters): void { + this.decorators.push(options); + } + + public getCollators(): RegisterCollatorParameters[] { + return this.collators; + } + + public getDecorators(): RegisterDecoratorParameters[] { + return this.decorators; + } +} + +class SearchEngineRegistry implements SearchEngineRegistryExtensionPoint { + private searchEngine: SearchEngine | null = null; + + public setSearchEngine(searchEngine: SearchEngine): void { + if (this.searchEngine) { + throw new Error('Multiple Search engines is not supported at this time'); + } + this.searchEngine = searchEngine; + } + + public getSearchEngine(): SearchEngine | null { + return this.searchEngine; + } +} + +/** + * The Search plugin is responsible for starting search indexing processes and return search results. + * @alpha + */ +export const searchPlugin = createBackendPlugin({ + pluginId: 'search', + register(env) { + const searchIndexRegistry = new SearchIndexRegistry(); + env.registerExtensionPoint( + searchIndexRegistryExtensionPoint, + searchIndexRegistry, + ); + + const searchEngineRegistry = new SearchEngineRegistry(); + env.registerExtensionPoint( + searchEngineRegistryExtensionPoint, + searchEngineRegistry, + ); + + env.registerInit({ + deps: { + logger: coreServices.logger, + config: coreServices.config, + permissions: coreServices.permissions, + http: coreServices.httpRouter, + searchIndexService: searchIndexServiceRef, + }, + async init({ config, logger, permissions, http, searchIndexService }) { + let searchEngine = searchEngineRegistry.getSearchEngine(); + if (!searchEngine) { + searchEngine = new LunrSearchEngine({ + logger: loggerToWinstonLogger(logger), + }); + } + + const collators = searchIndexRegistry.getCollators(); + const decorators = searchIndexRegistry.getDecorators(); + + await searchIndexService.start({ + searchEngine, + collators, + decorators, + }); + + const router = await createRouter({ + config, + permissions, + logger: loggerToWinstonLogger(logger), + engine: searchEngine, + types: searchIndexService.getDocumentTypes(), + }); + + http.use(router); + }, + }); + }, +}); diff --git a/plugins/search-backend/src/plugin.ts b/plugins/search-backend/src/plugin.ts deleted file mode 100644 index 7c01ddc607..0000000000 --- a/plugins/search-backend/src/plugin.ts +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - coreServices, - createBackendPlugin, -} from '@backstage/backend-plugin-api'; -import { loggerToWinstonLogger } from '@backstage/backend-common'; -import { - RegisterCollatorParameters, - RegisterDecoratorParameters, - LunrSearchEngine, -} from '@backstage/plugin-search-backend-node'; -import { - searchIndexBuilderService, - searchIndexRegistryExtensionPoint, - SearchIndexRegistryExtensionPoint, - SearchEngineRegistryExtensionPoint, - searchEngineRegistryExtensionPoint, -} from '@backstage/plugin-search-backend-node/alpha'; - -import { createRouter } from './service/router'; -import { SearchEngine } from '@backstage/plugin-search-common'; - -class SearchIndexRegistry implements SearchIndexRegistryExtensionPoint { - private collators: RegisterCollatorParameters[] = []; - private decorators: RegisterDecoratorParameters[] = []; - - public addCollator(options: RegisterCollatorParameters): void { - this.collators.push(options); - } - - public addDecorator(options: RegisterDecoratorParameters): void { - this.decorators.push(options); - } - - public getCollators(): RegisterCollatorParameters[] { - return this.collators; - } - - public getDecorators(): RegisterDecoratorParameters[] { - return this.decorators; - } -} - -class SearchEngineRegistry implements SearchEngineRegistryExtensionPoint { - private searchEngine: SearchEngine | null = null; - - public setSearchEngine(searchEngine: SearchEngine): void { - if (this.searchEngine) { - throw new Error('Multiple Search engines is not supported at this time'); - } - this.searchEngine = searchEngine; - } - - public getSearchEngine(): SearchEngine | null { - return this.searchEngine; - } -} - -export const searchPlugin = createBackendPlugin({ - pluginId: 'search', - register(env) { - const searchIndexRegistry = new SearchIndexRegistry(); - env.registerExtensionPoint( - searchIndexRegistryExtensionPoint, - searchIndexRegistry, - ); - - const searchEngineRegistry = new SearchEngineRegistry(); - env.registerExtensionPoint( - searchEngineRegistryExtensionPoint, - searchEngineRegistry, - ); - - env.registerInit({ - deps: { - logger: coreServices.logger, - config: coreServices.config, - permissions: coreServices.permissions, - http: coreServices.httpRouter, - searchIndexBuilder: searchIndexBuilderService, - }, - async init({ config, logger, permissions, http, searchIndexBuilder }) { - let searchEngine = searchEngineRegistry.getSearchEngine(); - if (!searchEngine) { - searchEngine = new LunrSearchEngine({ - logger: loggerToWinstonLogger(logger), - }); - } - - const collators = searchIndexRegistry.getCollators(); - const decorators = searchIndexRegistry.getDecorators(); - - const { scheduler } = await searchIndexBuilder.build({ - searchEngine, - collators, - decorators, - }); - scheduler.start(); - - const router = await createRouter({ - config, - permissions, - logger: loggerToWinstonLogger(logger), - engine: searchEngine, - types: searchIndexBuilder.getDocumentTypes(), - }); - - http.use(router); - }, - }); - }, -}); From 1469daa409e313f4ecb966ec773aab10df81ef0a Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 6 Mar 2023 15:27:33 +0100 Subject: [PATCH 13/27] add changesets and how to guide Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- .changeset/funny-pots-hide.md | 5 ++ .changeset/giant-hairs-serve.md | 6 ++ .changeset/nine-clouds-flow.md | 5 ++ .changeset/warm-buses-switch.md | 5 ++ docs/features/search/how-to-guides.md | 82 +++++++++++++++++++++++++++ 5 files changed, 103 insertions(+) create mode 100644 .changeset/funny-pots-hide.md create mode 100644 .changeset/giant-hairs-serve.md create mode 100644 .changeset/nine-clouds-flow.md create mode 100644 .changeset/warm-buses-switch.md diff --git a/.changeset/funny-pots-hide.md b/.changeset/funny-pots-hide.md new file mode 100644 index 0000000000..32388a93a5 --- /dev/null +++ b/.changeset/funny-pots-hide.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend': minor +--- + +Exports search plugin that can be used with the new backend system. For documentation on how to migrate, check out the [how to migrate to the new backend system guide](../docs/features/search/how-to-guides.md#how-to-migrate-to-use-search-together-with-the-new-backend-system). diff --git a/.changeset/giant-hairs-serve.md b/.changeset/giant-hairs-serve.md new file mode 100644 index 0000000000..ae450234d1 --- /dev/null +++ b/.changeset/giant-hairs-serve.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +'@backstage/plugin-search-backend-module-pg': minor +--- + +Search backend modules migrated to the new backend system. For documentation on how to migrate, check out the [how to migrate to the new backend system guide](../docs/features/search/how-to-guides.md#how-to-migrate-to-use-search-together-with-the-new-backend-system). diff --git a/.changeset/nine-clouds-flow.md b/.changeset/nine-clouds-flow.md new file mode 100644 index 0000000000..fca6bbbf1c --- /dev/null +++ b/.changeset/nine-clouds-flow.md @@ -0,0 +1,5 @@ +--- +'example-backend-next': patch +--- + +Adds search plugin and search index registry to backend. diff --git a/.changeset/warm-buses-switch.md b/.changeset/warm-buses-switch.md new file mode 100644 index 0000000000..5564f33d67 --- /dev/null +++ b/.changeset/warm-buses-switch.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-node': minor +--- + +Exports services and extension points that can be used with the new backend system. For documentation on how to migrate, check out the [how to migrate to the new backend system guide](../docs/features/search/how-to-guides.md#how-to-migrate-to-use-search-together-with-the-new-backend-system). diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index 5b28e259b5..e8f58e9f26 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -369,3 +369,85 @@ export const SearchModal = ({ toggleModal }: { toggleModal: () => void }) => ( ``` There are other more specific search results layout components that also accept result item extensions, check their documentation: [SearchResultList](https://backstage.io/storybook/?path=/story/plugins-search-searchresultlist--with-result-item-extensions) and [SearchResultGroup](https://backstage.io/storybook/?path=/story/plugins-search-searchresultgroup--with-result-item-extensions). + +## How to migrate to use Search together with the new backend system + +> DISCLAIMER: The new backend system is in alpha, and so are the search backend support for the new backend system. We don't recommend you to migrate your backend installations to the new system yet. But if you want to experiment, this is the guide for you! + +Recently, the Backstage maintainers [announced the new Backend System](https://backstage.io/blog/2023/02/15/backend-system-alpha). The search plugins are now migrated to support the new backend system. In this guide you will learn how to update your backend set up. + +1. In packages/backend/search.ts + +```ts +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; +import { ToolDocumentCollatorFactory } from '@backstage/plugin-explore-backend'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; + +export const searchIndexRegistry = createBackendModule({ + moduleId: 'searchIndexRegistry', + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + indexRegistry: searchIndexRegistryExtensionPoint, + config: coreServices.config, + discovery: coreServices.discovery, + tokenManager: coreServices.tokenManager, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + }, + async init({ + indexRegistry, + config, + logger, + discovery, + tokenManager, + scheduler, + }) { + // define scheule, the same way you did it before + const schedule = scheduler.createScheduledTaskRunner({ + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + // A 3 second delay gives the backend server a chance to initialize before + // any collators are executed, which may attempt requests against the API. + initialDelay: { seconds: 3 }, + }); + + indexRegistry.addCollator({ + schedule, + factory: DefaultCatalogCollatorFactory.fromConfig(config, { + discovery, + tokenManager, + }), + }); + + // .... other collators and decorators + }, + }); + }, +}); +``` + +2. In packages/backend/index.ts + +```ts +import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; +import { elasticSearchEngineModule } from '@backstage/plugin-search-backend-module-elasticsearch/alpha'; +import { searchIndexRegistry } from './plugins/search'; + +const backend = createBackend(); +// ...other modules +backend.add(searchPlugin()); +backend.add(searchIndexRegistry()); + +// the default search engine is Lunr, if you want to extend the search backend with another search engine. +backend.add(elasticSearchEngineModule()); + +backend.start(); +``` From 6422b85d468abc1c4cc9506ec18262f1f72ad04f Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 6 Mar 2023 15:47:45 +0100 Subject: [PATCH 14/27] fixup api report Signed-off-by: Emma Indal --- plugins/search-backend-node/alpha-api-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-backend-node/alpha-api-report.md b/plugins/search-backend-node/alpha-api-report.md index f53f86b709..27d4875bd6 100644 --- a/plugins/search-backend-node/alpha-api-report.md +++ b/plugins/search-backend-node/alpha-api-report.md @@ -56,7 +56,7 @@ export interface SearchIndexService { } // @alpha -export const searchIndexService: ServiceRef; +export const searchIndexServiceRef: ServiceRef; // (No @packageDocumentation comment for this package) ``` From 1473dd9eb69cbe0f60b23beddc7de3b0fc5d4afa Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Tue, 7 Mar 2023 09:22:22 +0100 Subject: [PATCH 15/27] add initial test for search plugin Signed-off-by: Emma Indal --- plugins/search-backend/package.json | 1 + plugins/search-backend/src/alpha.test.ts | 31 ++++++++++++++++++++++++ yarn.lock | 1 + 3 files changed, 33 insertions(+) create mode 100644 plugins/search-backend/src/alpha.test.ts diff --git a/plugins/search-backend/package.json b/plugins/search-backend/package.json index 6e971ba646..50a28d3ac5 100644 --- a/plugins/search-backend/package.json +++ b/plugins/search-backend/package.json @@ -57,6 +57,7 @@ "zod": "~3.18.0" }, "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@types/supertest": "^2.0.8", "supertest": "^6.1.3" diff --git a/plugins/search-backend/src/alpha.test.ts b/plugins/search-backend/src/alpha.test.ts new file mode 100644 index 0000000000..527f099236 --- /dev/null +++ b/plugins/search-backend/src/alpha.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { startTestBackend } from '@backstage/backend-test-utils'; +import request from 'supertest'; +import { searchPlugin } from './alpha'; + +describe('searchPlugin', () => { + it('should serve search results on query endpoint', async () => { + const { server } = await startTestBackend({ + features: [searchPlugin()], + }); + + const response = await request(server).get('/api/search/query'); + expect(response.status).toBe(200); + expect(response.body).toEqual({ numberOfResults: 0, results: [] }); + }); +}); diff --git a/yarn.lock b/yarn.lock index c2c60036fb..b5203fbe29 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7837,6 +7837,7 @@ __metadata: dependencies: "@backstage/backend-common": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/errors": "workspace:^" From 1adc2c787e7ddb04c31d2c74abd18d5c772bacf3 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Tue, 7 Mar 2023 13:53:58 +0100 Subject: [PATCH 16/27] feat(search): create search collator modules Co-authored-by: Emma Indal Signed-off-by: Camila Belo --- packages/backend-next/package.json | 4 +- packages/backend-next/src/index.ts | 28 ++++- packages/backend-next/src/plugins/search.ts | 82 -------------- plugins/catalog-backend/api-report.md | 46 +++----- plugins/catalog-backend/package.json | 1 + plugins/catalog-backend/src/index.ts | 14 +++ plugins/catalog-backend/src/search/index.ts | 5 - plugins/explore-backend/api-report.md | 34 +----- plugins/explore-backend/package.json | 1 + plugins/explore-backend/src/index.ts | 16 ++- .../.eslintrc.js | 1 + .../CHANGELOG.md | 1 + .../search-backend-module-catalog/README.md | 7 ++ .../alpha-api-report.md | 66 +++++++++++ .../api-report.md | 52 +++++++++ .../package.json | 70 ++++++++++++ .../src/alpha.ts | 90 +++++++++++++++ .../CatalogCollatorEntityTransformer.ts | 0 .../DefaultCatalogCollatorFactory.test.ts | 0 .../DefaultCatalogCollatorFactory.ts | 0 ...ltCatalogCollatorEntityTransformer.test.ts | 0 ...defaultCatalogCollatorEntityTransformer.ts | 0 .../src/collators/index.ts | 21 ++++ .../src/index.ts | 22 ++++ .../alpha-api-report.md | 22 ++-- .../src/alpha.ts | 13 ++- .../.eslintrc.js | 1 + .../CHANGELOG.md | 1 + .../search-backend-module-explore/README.md | 7 ++ .../alpha-api-report.md | 56 ++++++++++ .../api-report.md | 39 +++++++ .../package.json | 71 ++++++++++++ .../src/alpha.ts | 85 ++++++++++++++ .../collators}/ToolDocumentCollatorFactory.ts | 0 .../src/collators}/index.ts | 1 + .../src/index.ts | 22 ++++ .../alpha-api-report.md | 2 +- plugins/search-backend-module-pg/src/alpha.ts | 4 +- .../.eslintrc.js | 1 + .../CHANGELOG.md | 1 + .../search-backend-module-techdocs/README.md | 7 ++ .../alpha-api-report.md | 40 +++++++ .../api-report.md | 42 +++++++ .../package.json | 71 ++++++++++++ .../src/alpha.ts | 89 +++++++++++++++ .../DefaultTechDocsCollatorFactory.test.ts | 0 .../DefaultTechDocsCollatorFactory.ts | 0 .../src/collators/index.ts | 19 ++++ .../src/index.ts | 22 ++++ plugins/techdocs-backend/api-report.md | 32 +----- plugins/techdocs-backend/package.json | 1 + plugins/techdocs-backend/src/search/index.ts | 15 ++- yarn.lock | 105 +++++++++++++++++- 53 files changed, 1124 insertions(+), 206 deletions(-) delete mode 100644 packages/backend-next/src/plugins/search.ts create mode 100644 plugins/search-backend-module-catalog/.eslintrc.js create mode 100644 plugins/search-backend-module-catalog/CHANGELOG.md create mode 100644 plugins/search-backend-module-catalog/README.md create mode 100644 plugins/search-backend-module-catalog/alpha-api-report.md create mode 100644 plugins/search-backend-module-catalog/api-report.md create mode 100644 plugins/search-backend-module-catalog/package.json create mode 100644 plugins/search-backend-module-catalog/src/alpha.ts rename plugins/{catalog-backend/src/search => search-backend-module-catalog/src/collators}/CatalogCollatorEntityTransformer.ts (100%) rename plugins/{catalog-backend/src/search => search-backend-module-catalog/src/collators}/DefaultCatalogCollatorFactory.test.ts (100%) rename plugins/{catalog-backend/src/search => search-backend-module-catalog/src/collators}/DefaultCatalogCollatorFactory.ts (100%) rename plugins/{catalog-backend/src/search => search-backend-module-catalog/src/collators}/defaultCatalogCollatorEntityTransformer.test.ts (100%) rename plugins/{catalog-backend/src/search => search-backend-module-catalog/src/collators}/defaultCatalogCollatorEntityTransformer.ts (100%) create mode 100644 plugins/search-backend-module-catalog/src/collators/index.ts create mode 100644 plugins/search-backend-module-catalog/src/index.ts create mode 100644 plugins/search-backend-module-explore/.eslintrc.js create mode 100644 plugins/search-backend-module-explore/CHANGELOG.md create mode 100644 plugins/search-backend-module-explore/README.md create mode 100644 plugins/search-backend-module-explore/alpha-api-report.md create mode 100644 plugins/search-backend-module-explore/api-report.md create mode 100644 plugins/search-backend-module-explore/package.json create mode 100644 plugins/search-backend-module-explore/src/alpha.ts rename plugins/{explore-backend/src/search => search-backend-module-explore/src/collators}/ToolDocumentCollatorFactory.ts (100%) rename plugins/{explore-backend/src/search => search-backend-module-explore/src/collators}/index.ts (99%) create mode 100644 plugins/search-backend-module-explore/src/index.ts create mode 100644 plugins/search-backend-module-techdocs/.eslintrc.js create mode 100644 plugins/search-backend-module-techdocs/CHANGELOG.md create mode 100644 plugins/search-backend-module-techdocs/README.md create mode 100644 plugins/search-backend-module-techdocs/alpha-api-report.md create mode 100644 plugins/search-backend-module-techdocs/api-report.md create mode 100644 plugins/search-backend-module-techdocs/package.json create mode 100644 plugins/search-backend-module-techdocs/src/alpha.ts rename plugins/{techdocs-backend/src/search => search-backend-module-techdocs/src/collators}/DefaultTechDocsCollatorFactory.test.ts (100%) rename plugins/{techdocs-backend/src/search => search-backend-module-techdocs/src/collators}/DefaultTechDocsCollatorFactory.ts (100%) create mode 100644 plugins/search-backend-module-techdocs/src/collators/index.ts create mode 100644 plugins/search-backend-module-techdocs/src/index.ts diff --git a/packages/backend-next/package.json b/packages/backend-next/package.json index 8df5ace282..9859717fd5 100644 --- a/packages/backend-next/package.json +++ b/packages/backend-next/package.json @@ -30,9 +30,11 @@ "@backstage/backend-plugin-api": "workspace:^", "@backstage/plugin-app-backend": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", - "@backstage/plugin-explore-backend": "workspace:^", "@backstage/plugin-scaffolder-backend": "workspace:^", "@backstage/plugin-search-backend": "workspace:^", + "@backstage/plugin-search-backend-module-catalog": "workspace:^", + "@backstage/plugin-search-backend-module-explore": "workspace:^", + "@backstage/plugin-search-backend-module-techdocs": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-techdocs-backend": "workspace:^", "@backstage/plugin-todo-backend": "workspace:^" diff --git a/packages/backend-next/src/index.ts b/packages/backend-next/src/index.ts index 10680ca798..231343203a 100644 --- a/packages/backend-next/src/index.ts +++ b/packages/backend-next/src/index.ts @@ -14,24 +14,40 @@ * limitations under the License. */ -import { catalogPlugin } from '@backstage/plugin-catalog-backend/alpha'; -import { catalogModuleTemplateKind } from '@backstage/plugin-scaffolder-backend/alpha'; import { createBackend } from '@backstage/backend-defaults'; import { appPlugin } from '@backstage/plugin-app-backend/alpha'; import { todoPlugin } from '@backstage/plugin-todo-backend'; import { techdocsPlugin } from '@backstage/plugin-techdocs-backend/alpha'; +import { catalogPlugin } from '@backstage/plugin-catalog-backend/alpha'; +import { catalogModuleTemplateKind } from '@backstage/plugin-scaffolder-backend/alpha'; import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; -import { searchIndexRegistry } from './plugins/search'; +import { searchModuleCatalogCollator } from '@backstage/plugin-search-backend-module-catalog/alpha'; +import { searchModuleTechDocsCollator } from '@backstage/plugin-search-backend-module-techdocs/alpha'; +import { searchModuleExploreCollator } from '@backstage/plugin-search-backend-module-explore/alpha'; const backend = createBackend(); -backend.add(catalogPlugin()); -backend.add(catalogModuleTemplateKind()); backend.add(appPlugin({ appPackageName: 'example-app' })); + +// Todo backend.add(todoPlugin()); + +// Techdocs backend.add(techdocsPlugin()); +// Catalog +backend.add(catalogPlugin()); +backend.add(catalogModuleTemplateKind()); + +// Search +const schedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, +}; backend.add(searchPlugin()); -backend.add(searchIndexRegistry()); +backend.add(searchModuleCatalogCollator({ schedule })); +backend.add(searchModuleTechDocsCollator({ schedule })); +backend.add(searchModuleExploreCollator({ schedule })); backend.start(); diff --git a/packages/backend-next/src/plugins/search.ts b/packages/backend-next/src/plugins/search.ts deleted file mode 100644 index 1329135e56..0000000000 --- a/packages/backend-next/src/plugins/search.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - coreServices, - createBackendModule, -} from '@backstage/backend-plugin-api'; -import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; -import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; -import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; -import { ToolDocumentCollatorFactory } from '@backstage/plugin-explore-backend'; -import { loggerToWinstonLogger } from '@backstage/backend-common'; - -export const searchIndexRegistry = createBackendModule({ - moduleId: 'searchIndexRegistry', - pluginId: 'search', - register(env) { - env.registerInit({ - deps: { - indexRegistry: searchIndexRegistryExtensionPoint, - config: coreServices.config, - discovery: coreServices.discovery, - tokenManager: coreServices.tokenManager, - logger: coreServices.logger, - scheduler: coreServices.scheduler, - }, - async init({ - indexRegistry, - config, - logger, - discovery, - tokenManager, - scheduler, - }) { - const schedule = scheduler.createScheduledTaskRunner({ - frequency: { minutes: 10 }, - timeout: { minutes: 15 }, - // A 3 second delay gives the backend server a chance to initialize before - // any collators are executed, which may attempt requests against the API. - initialDelay: { seconds: 3 }, - }); - - indexRegistry.addCollator({ - schedule, - factory: DefaultCatalogCollatorFactory.fromConfig(config, { - discovery, - tokenManager, - }), - }); - - indexRegistry.addCollator({ - schedule, - factory: DefaultTechDocsCollatorFactory.fromConfig(config, { - discovery, - logger: loggerToWinstonLogger(logger), - tokenManager, - }), - }); - - indexRegistry.addCollator({ - schedule, - factory: ToolDocumentCollatorFactory.fromConfig(config, { - discovery, - logger: loggerToWinstonLogger(logger), - }), - }); - }, - }); - }, -}); diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 5e7b2dee02..3d6a33fbf9 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -11,6 +11,7 @@ import { AnalyzeLocationGenerateEntity as AnalyzeLocationGenerateEntity_2 } from import { AnalyzeLocationRequest as AnalyzeLocationRequest_2 } from '@backstage/plugin-catalog-common'; import { AnalyzeLocationResponse as AnalyzeLocationResponse_2 } from '@backstage/plugin-catalog-common'; import { CatalogApi } from '@backstage/catalog-client'; +import { CatalogCollatorEntityTransformer } from '@backstage/plugin-search-backend-module-catalog'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; import { CatalogProcessor as CatalogProcessor_2 } from '@backstage/plugin-catalog-node'; import { CatalogProcessorCache as CatalogProcessorCache_2 } from '@backstage/plugin-catalog-node'; @@ -23,8 +24,15 @@ import { CatalogProcessorRefreshKeysResult as CatalogProcessorRefreshKeysResult_ import { CatalogProcessorRelationResult as CatalogProcessorRelationResult_2 } from '@backstage/plugin-catalog-node'; import { CatalogProcessorResult as CatalogProcessorResult_2 } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; +<<<<<<< HEAD import { DeferredEntity as DeferredEntity_2 } from '@backstage/plugin-catalog-node'; import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +======= +import { defaultCatalogCollatorEntityTransformer } from '@backstage/plugin-search-backend-module-catalog'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-search-backend-module-catalog'; +import { DefaultCatalogCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-catalog'; +import { DeferredEntity } from '@backstage/plugin-catalog-node'; +>>>>>>> 56a45401d3d (feat(search): create search collator modules) import { Entity } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import { EntityProvider as EntityProvider_2 } from '@backstage/plugin-catalog-node'; @@ -44,7 +52,11 @@ import { PermissionRule } from '@backstage/plugin-permission-node'; import { PermissionRuleParams } from '@backstage/plugin-permission-common'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; +<<<<<<< HEAD import { Readable } from 'stream'; +======= +import { processingResult } from '@backstage/plugin-catalog-node'; +>>>>>>> 56a45401d3d (feat(search): create search collator modules) import { Router } from 'express'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { TokenManager } from '@backstage/backend-common'; @@ -160,10 +172,7 @@ export class CatalogBuilder { useLegacySingleProcessorValidation(): this; } -// @public (undocumented) -export type CatalogCollatorEntityTransformer = ( - entity: Entity, -) => Omit; +export { CatalogCollatorEntityTransformer }; // @public (undocumented) export type CatalogEnvironment = { @@ -286,34 +295,11 @@ export class DefaultCatalogCollator { readonly visibilityPermission: Permission; } -// @public (undocumented) -export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer; +export { defaultCatalogCollatorEntityTransformer }; -// @public (undocumented) -export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { - // (undocumented) - static fromConfig( - _config: Config, - options: DefaultCatalogCollatorFactoryOptions, - ): DefaultCatalogCollatorFactory; - // (undocumented) - getCollator(): Promise; - // (undocumented) - readonly type = 'software-catalog'; - // (undocumented) - readonly visibilityPermission: Permission; -} +export { DefaultCatalogCollatorFactory }; -// @public (undocumented) -export type DefaultCatalogCollatorFactoryOptions = { - discovery: PluginEndpointDiscovery; - tokenManager: TokenManager; - locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; - batchSize?: number; - catalogClient?: CatalogApi; - entityTransformer?: CatalogCollatorEntityTransformer; -}; +export { DefaultCatalogCollatorFactoryOptions }; // @public @deprecated (undocumented) export type DeferredEntity = DeferredEntity_2; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 1e837fd9ba..203a89b730 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -57,6 +57,7 @@ "@backstage/plugin-permission-common": "workspace:^", "@backstage/plugin-permission-node": "workspace:^", "@backstage/plugin-scaffolder-common": "workspace:^", + "@backstage/plugin-search-backend-module-catalog": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/types": "workspace:^", "@opentelemetry/api": "^1.3.0", diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index b3d2bc95c7..594b96b347 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -27,3 +27,17 @@ export * from './processing'; export * from './search'; export * from './service'; export * from './deprecated'; + +import { + DefaultCatalogCollatorFactory as _DefaultCatalogCollatorFactory, + defaultCatalogCollatorEntityTransformer as _defaultCatalogCollatorEntityTransformer, +} from '@backstage/plugin-search-backend-module-catalog'; + +/** + * @deprecated + * import from @backstage/search-backend-module-catalog instead + */ +export type { + DefaultCatalogCollatorFactoryOptions, + CatalogCollatorEntityTransformer, +} from '@backstage/plugin-search-backend-module-catalog'; diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 3603ab2e60..50966e4c5b 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -14,11 +14,6 @@ * limitations under the License. */ -export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; -export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; -export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; -export { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; - /** * todo(backstage/techdocs-core): stop exporting this in a future release. */ diff --git a/plugins/explore-backend/api-report.md b/plugins/explore-backend/api-report.md index cc112ed3cc..4bcdc07079 100644 --- a/plugins/explore-backend/api-report.md +++ b/plugins/explore-backend/api-report.md @@ -3,18 +3,14 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -/// - -import { Config } from '@backstage/config'; -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; import { ExploreTool } from '@backstage/plugin-explore-common'; import express from 'express'; import { GetExploreToolsRequest } from '@backstage/plugin-explore-common'; import { GetExploreToolsResponse } from '@backstage/plugin-explore-common'; -import { IndexableDocument } from '@backstage/plugin-search-common'; import { Logger } from 'winston'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; -import { Readable } from 'stream'; +import { ToolDocument } from '@backstage/plugin-search-backend-module-explore'; +import { ToolDocumentCollatorFactory } from '@backstage/plugin-search-backend-module-explore'; +import { ToolDocumentCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-explore'; // @public (undocumented) export function createRouter(options: RouterOptions): Promise; @@ -40,27 +36,9 @@ export class StaticExploreToolProvider implements ExploreToolProvider { getTools(request: GetExploreToolsRequest): Promise; } -// @public -export interface ToolDocument extends IndexableDocument, ExploreTool {} +export { ToolDocument }; -// @public -export class ToolDocumentCollatorFactory implements DocumentCollatorFactory { - // (undocumented) - execute(): AsyncGenerator; - // (undocumented) - static fromConfig( - _config: Config, - options: ToolDocumentCollatorFactoryOptions, - ): ToolDocumentCollatorFactory; - // (undocumented) - getCollator(): Promise; - // (undocumented) - readonly type: string; -} +export { ToolDocumentCollatorFactory }; -// @public -export type ToolDocumentCollatorFactoryOptions = { - discovery: PluginEndpointDiscovery; - logger: Logger; -}; +export { ToolDocumentCollatorFactoryOptions }; ``` diff --git a/plugins/explore-backend/package.json b/plugins/explore-backend/package.json index 00629c8da4..ce1a22e502 100644 --- a/plugins/explore-backend/package.json +++ b/plugins/explore-backend/package.json @@ -25,6 +25,7 @@ "@backstage/backend-common": "workspace:^", "@backstage/config": "workspace:^", "@backstage/plugin-explore-common": "workspace:^", + "@backstage/plugin-search-backend-module-explore": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@types/express": "*", "express": "^4.18.1", diff --git a/plugins/explore-backend/src/index.ts b/plugins/explore-backend/src/index.ts index a0e7216621..6c456bffc2 100644 --- a/plugins/explore-backend/src/index.ts +++ b/plugins/explore-backend/src/index.ts @@ -20,7 +20,6 @@ * @packageDocumentation */ -export * from './search'; export * from './service'; export * from './tools'; @@ -28,3 +27,18 @@ export * from './tools'; * @internal Example only - do not use in production */ export { exampleTools } from './example/exampleTools'; + +/** + * @deprecated + * import from @backstage/plugin-search-backend-module-explore instead + */ +export { ToolDocumentCollatorFactory } from '@backstage/plugin-search-backend-module-explore'; + +/** + * @deprecated + * import from @backstage/plugin-search-backend-module-explore instead + */ +export type { + ToolDocument, + ToolDocumentCollatorFactoryOptions, +} from '@backstage/plugin-search-backend-module-explore'; diff --git a/plugins/search-backend-module-catalog/.eslintrc.js b/plugins/search-backend-module-catalog/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/search-backend-module-catalog/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/search-backend-module-catalog/CHANGELOG.md b/plugins/search-backend-module-catalog/CHANGELOG.md new file mode 100644 index 0000000000..c870819424 --- /dev/null +++ b/plugins/search-backend-module-catalog/CHANGELOG.md @@ -0,0 +1 @@ +# @backstage/plugin-search-backend-module-catalog diff --git a/plugins/search-backend-module-catalog/README.md b/plugins/search-backend-module-catalog/README.md new file mode 100644 index 0000000000..9860f86b1d --- /dev/null +++ b/plugins/search-backend-module-catalog/README.md @@ -0,0 +1,7 @@ +# search-backend-module-catalog + +... + +## Getting started + +... diff --git a/plugins/search-backend-module-catalog/alpha-api-report.md b/plugins/search-backend-module-catalog/alpha-api-report.md new file mode 100644 index 0000000000..e0173fce74 --- /dev/null +++ b/plugins/search-backend-module-catalog/alpha-api-report.md @@ -0,0 +1,66 @@ +## API Report File for "@backstage/plugin-search-backend-module-catalog" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +/// + +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { CatalogApi } from '@backstage/catalog-client'; +import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; +import { Config } from '@backstage/config'; +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { Entity } from '@backstage/catalog-model'; +import { GetEntitiesRequest } from '@backstage/catalog-client'; +import { Permission } from '@backstage/plugin-permission-common'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Readable } from 'stream'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { TokenManager } from '@backstage/backend-common'; + +// @public (undocumented) +export type CatalogCollatorEntityTransformer = ( + entity: Entity, +) => Omit; + +// @public (undocumented) +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + _config: Config, + options: DefaultCatalogCollatorFactoryOptions, + ): DefaultCatalogCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type = 'software-catalog'; + // (undocumented) + readonly visibilityPermission: Permission; +} + +// @public (undocumented) +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; + entityTransformer?: CatalogCollatorEntityTransformer; +}; + +// @alpha +export const searchModuleCatalogCollator: ( + options: SearchModuleCatalogCollatorOptions, +) => BackendFeature; + +// @alpha +export type SearchModuleCatalogCollatorOptions = Omit< + DefaultCatalogCollatorFactoryOptions, + 'discovery' | 'tokenManager' +> & { + schedule: TaskScheduleDefinition; +}; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/search-backend-module-catalog/api-report.md b/plugins/search-backend-module-catalog/api-report.md new file mode 100644 index 0000000000..b773860fe8 --- /dev/null +++ b/plugins/search-backend-module-catalog/api-report.md @@ -0,0 +1,52 @@ +## API Report File for "@backstage/plugin-search-backend-module-catalog" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +/// + +import { CatalogApi } from '@backstage/catalog-client'; +import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; +import { Config } from '@backstage/config'; +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { Entity } from '@backstage/catalog-model'; +import { GetEntitiesRequest } from '@backstage/catalog-client'; +import { Permission } from '@backstage/plugin-permission-common'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Readable } from 'stream'; +import { TokenManager } from '@backstage/backend-common'; + +// @public (undocumented) +export type CatalogCollatorEntityTransformer = ( + entity: Entity, +) => Omit; + +// @public (undocumented) +export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer; + +// @public (undocumented) +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + _config: Config, + options: DefaultCatalogCollatorFactoryOptions, + ): DefaultCatalogCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type = 'software-catalog'; + // (undocumented) + readonly visibilityPermission: Permission; +} + +// @public (undocumented) +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; + entityTransformer?: CatalogCollatorEntityTransformer; +}; +``` diff --git a/plugins/search-backend-module-catalog/package.json b/plugins/search-backend-module-catalog/package.json new file mode 100644 index 0000000000..6c7b78437c --- /dev/null +++ b/plugins/search-backend-module-catalog/package.json @@ -0,0 +1,70 @@ +{ + "name": "@backstage/plugin-search-backend-module-catalog", + "description": "A module for the search backend that exports catalog modules", + "version": "1.1.4-next.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } + }, + "backstage": { + "role": "backend-plugin-module" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-client": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", + "@backstage/plugin-permission-common": "workspace:^", + "@backstage/plugin-search-backend-node": "workspace:^", + "@backstage/plugin-search-common": "workspace:^", + "@elastic/elasticsearch": "^7.13.0", + "@opensearch-project/opensearch": "^2.0.0", + "aws-os-connection": "^0.2.0", + "aws-sdk": "^2.948.0", + "elastic-builder": "^2.16.0", + "lodash": "^4.17.21", + "uuid": "^8.3.2", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@elastic/elasticsearch-mock": "^1.0.0", + "@short.io/opensearch-mock": "^0.3.1", + "msw": "^1.0.0" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/search-backend-module-catalog/src/alpha.ts b/plugins/search-backend-module-catalog/src/alpha.ts new file mode 100644 index 0000000000..d4530b582b --- /dev/null +++ b/plugins/search-backend-module-catalog/src/alpha.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @packageDocumentation + * A module for the search backend that exports Catalog modules. + */ + +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; + +import { + CatalogCollatorEntityTransformer, + DefaultCatalogCollatorFactory, + DefaultCatalogCollatorFactoryOptions, +} from './collators'; + +export type { + CatalogCollatorEntityTransformer, + DefaultCatalogCollatorFactory, + DefaultCatalogCollatorFactoryOptions, +}; + +/** + * @alpha + * Options for {@link searchModuleCatalogCollator}. + */ +export type SearchModuleCatalogCollatorOptions = Omit< + DefaultCatalogCollatorFactoryOptions, + 'discovery' | 'tokenManager' +> & { + schedule: TaskScheduleDefinition; +}; + +/** + * @alpha + * Search backend module for the Catalog index. + */ +export const searchModuleCatalogCollator = createBackendModule( + (options: SearchModuleCatalogCollatorOptions) => ({ + moduleId: 'catalogCollator', + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + config: coreServices.config, + discovery: coreServices.discovery, + tokenManager: coreServices.tokenManager, + scheduler: coreServices.scheduler, + indexRegistry: searchIndexRegistryExtensionPoint, + }, + async init({ + config, + discovery, + tokenManager, + scheduler, + indexRegistry, + }) { + const { schedule, ...rest } = options; + + indexRegistry.addCollator({ + schedule: scheduler.createScheduledTaskRunner(schedule), + factory: DefaultCatalogCollatorFactory.fromConfig(config, { + ...rest, + discovery, + tokenManager, + }), + }); + }, + }); + }, + }), +); diff --git a/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts b/plugins/search-backend-module-catalog/src/collators/CatalogCollatorEntityTransformer.ts similarity index 100% rename from plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts rename to plugins/search-backend-module-catalog/src/collators/CatalogCollatorEntityTransformer.ts diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts b/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.test.ts similarity index 100% rename from plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts rename to plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.test.ts diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.ts similarity index 100% rename from plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts rename to plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.ts diff --git a/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts b/plugins/search-backend-module-catalog/src/collators/defaultCatalogCollatorEntityTransformer.test.ts similarity index 100% rename from plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts rename to plugins/search-backend-module-catalog/src/collators/defaultCatalogCollatorEntityTransformer.test.ts diff --git a/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts b/plugins/search-backend-module-catalog/src/collators/defaultCatalogCollatorEntityTransformer.ts similarity index 100% rename from plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts rename to plugins/search-backend-module-catalog/src/collators/defaultCatalogCollatorEntityTransformer.ts diff --git a/plugins/search-backend-module-catalog/src/collators/index.ts b/plugins/search-backend-module-catalog/src/collators/index.ts new file mode 100644 index 0000000000..5876e71a02 --- /dev/null +++ b/plugins/search-backend-module-catalog/src/collators/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; +export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; + +export { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; +export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; diff --git a/plugins/search-backend-module-catalog/src/index.ts b/plugins/search-backend-module-catalog/src/index.ts new file mode 100644 index 0000000000..7998562311 --- /dev/null +++ b/plugins/search-backend-module-catalog/src/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @packageDocumentation + * A module for the search backend that exports Catalog modules. + */ + +export * from './collators'; diff --git a/plugins/search-backend-module-elasticsearch/alpha-api-report.md b/plugins/search-backend-module-elasticsearch/alpha-api-report.md index acdd14e781..fca920f3d1 100644 --- a/plugins/search-backend-module-elasticsearch/alpha-api-report.md +++ b/plugins/search-backend-module-elasticsearch/alpha-api-report.md @@ -26,17 +26,6 @@ export type ElasticSearchCustomIndexTemplateBody = { template?: Record; }; -// @alpha -export const elasticSearchEngineModule: ( - options?: ElasticsearchEngineModuleOptions | undefined, -) => BackendFeature; - -// @alpha -export type ElasticsearchEngineModuleOptions = { - translator?: ElasticSearchQueryTranslator; - indexTemplate?: ElasticSearchCustomIndexTemplate; -}; - // @public (undocumented) export type ElasticSearchHighlightConfig = { fragmentDelimiter: string; @@ -57,5 +46,16 @@ export type ElasticSearchQueryTranslatorOptions = { highlightOptions?: ElasticSearchHighlightConfig; }; +// @alpha +export const searchModuleElasticsearchEngine: ( + options?: SearchModuleElasticsearchEngineOptions | undefined, +) => BackendFeature; + +// @alpha +export type SearchModuleElasticsearchEngineOptions = { + translator?: ElasticSearchQueryTranslator; + indexTemplate?: ElasticSearchCustomIndexTemplate; +}; + // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/search-backend-module-elasticsearch/src/alpha.ts b/plugins/search-backend-module-elasticsearch/src/alpha.ts index 8e75c9f603..d2ff1568f6 100644 --- a/plugins/search-backend-module-elasticsearch/src/alpha.ts +++ b/plugins/search-backend-module-elasticsearch/src/alpha.ts @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; import { coreServices, createBackendModule, } from '@backstage/backend-plugin-api'; import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; + import { ElasticSearchHighlightConfig, ElasticSearchQueryTranslatorOptions, @@ -31,9 +32,9 @@ import { /** * @alpha - * Options for {@link elasticSearchEngineModule}. + * Options for {@link searchModuleElasticsearchEngine}. */ -export type ElasticsearchEngineModuleOptions = { +export type SearchModuleElasticsearchEngineOptions = { translator?: ElasticSearchQueryTranslator; indexTemplate?: ElasticSearchCustomIndexTemplate; }; @@ -42,9 +43,9 @@ export type ElasticsearchEngineModuleOptions = { * @alpha * Search backend module for the Elasticsearch engine. */ -export const elasticSearchEngineModule = createBackendModule( - (options?: ElasticsearchEngineModuleOptions) => ({ - moduleId: 'elasticSearchEngineModule', +export const searchModuleElasticsearchEngine = createBackendModule( + (options?: SearchModuleElasticsearchEngineOptions) => ({ + moduleId: 'elasticsearchEngine', pluginId: 'search', register(env) { env.registerInit({ diff --git a/plugins/search-backend-module-explore/.eslintrc.js b/plugins/search-backend-module-explore/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/search-backend-module-explore/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/search-backend-module-explore/CHANGELOG.md b/plugins/search-backend-module-explore/CHANGELOG.md new file mode 100644 index 0000000000..8640d14b73 --- /dev/null +++ b/plugins/search-backend-module-explore/CHANGELOG.md @@ -0,0 +1 @@ +# @backstage/plugin-search-backend-module-explore diff --git a/plugins/search-backend-module-explore/README.md b/plugins/search-backend-module-explore/README.md new file mode 100644 index 0000000000..bbc8392f14 --- /dev/null +++ b/plugins/search-backend-module-explore/README.md @@ -0,0 +1,7 @@ +# search-backend-module-explore + +... + +## Getting started + +... diff --git a/plugins/search-backend-module-explore/alpha-api-report.md b/plugins/search-backend-module-explore/alpha-api-report.md new file mode 100644 index 0000000000..5c5189c8bd --- /dev/null +++ b/plugins/search-backend-module-explore/alpha-api-report.md @@ -0,0 +1,56 @@ +## API Report File for "@backstage/plugin-search-backend-module-explore" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +/// + +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { Config } from '@backstage/config'; +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { ExploreTool } from '@backstage/plugin-explore-common'; +import { IndexableDocument } from '@backstage/plugin-search-common'; +import { Logger } from 'winston'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Readable } from 'stream'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + +// @alpha +export const searchModuleExploreCollator: ( + options: SearchModuleExploreCollatorOptions, +) => BackendFeature; + +// @alpha +export type SearchModuleExploreCollatorOptions = Omit< + ToolDocumentCollatorFactoryOptions, + 'logger' | 'discovery' +> & { + schedule: TaskScheduleDefinition; +}; + +// @public +export interface ToolDocument extends IndexableDocument, ExploreTool {} + +// @public +export class ToolDocumentCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + execute(): AsyncGenerator; + // (undocumented) + static fromConfig( + _config: Config, + options: ToolDocumentCollatorFactoryOptions, + ): ToolDocumentCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; +} + +// @public +export type ToolDocumentCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger; +}; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/search-backend-module-explore/api-report.md b/plugins/search-backend-module-explore/api-report.md new file mode 100644 index 0000000000..80e8c0c06b --- /dev/null +++ b/plugins/search-backend-module-explore/api-report.md @@ -0,0 +1,39 @@ +## API Report File for "@backstage/plugin-search-backend-module-explore" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +/// + +import { Config } from '@backstage/config'; +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { ExploreTool } from '@backstage/plugin-explore-common'; +import { IndexableDocument } from '@backstage/plugin-search-common'; +import { Logger } from 'winston'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Readable } from 'stream'; + +// @public +export interface ToolDocument extends IndexableDocument, ExploreTool {} + +// @public +export class ToolDocumentCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + execute(): AsyncGenerator; + // (undocumented) + static fromConfig( + _config: Config, + options: ToolDocumentCollatorFactoryOptions, + ): ToolDocumentCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; +} + +// @public +export type ToolDocumentCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger; +}; +``` diff --git a/plugins/search-backend-module-explore/package.json b/plugins/search-backend-module-explore/package.json new file mode 100644 index 0000000000..356059f845 --- /dev/null +++ b/plugins/search-backend-module-explore/package.json @@ -0,0 +1,71 @@ +{ + "name": "@backstage/plugin-search-backend-module-explore", + "description": "A module for the search backend that exports explore modules", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } + }, + "backstage": { + "role": "backend-plugin-module" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-client": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", + "@backstage/integration": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", + "@backstage/plugin-explore-common": "workspace:^", + "@backstage/plugin-permission-common": "workspace:^", + "@backstage/plugin-search-backend-node": "workspace:^", + "@backstage/plugin-search-common": "workspace:^", + "@types/express": "^4.17.6", + "dockerode": "^3.3.1", + "express": "^4.17.1", + "express-promise-router": "^4.1.0", + "fs-extra": "10.1.0", + "knex": "^2.0.0", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7", + "p-limit": "^3.1.0", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "msw": "^1.0.0" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/search-backend-module-explore/src/alpha.ts b/plugins/search-backend-module-explore/src/alpha.ts new file mode 100644 index 0000000000..357bb4233e --- /dev/null +++ b/plugins/search-backend-module-explore/src/alpha.ts @@ -0,0 +1,85 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @packageDocumentation + * A module for the search backend that exports Explore modules. + */ + +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; + +import { + ToolDocument, + ToolDocumentCollatorFactory, + ToolDocumentCollatorFactoryOptions, +} from './collators'; + +export type { + ToolDocumentCollatorFactoryOptions, + ToolDocument, + ToolDocumentCollatorFactory, +}; + +/** + * @alpha + * Options for {@link searchModuleExploreCollator}. + */ +export type SearchModuleExploreCollatorOptions = Omit< + ToolDocumentCollatorFactoryOptions, + 'logger' | 'discovery' +> & { + schedule: TaskScheduleDefinition; +}; + +/** + * @alpha + * Search backend module for the Explore index. + */ +export const searchModuleExploreCollator = createBackendModule( + (options: SearchModuleExploreCollatorOptions) => ({ + moduleId: 'exploreCollator', + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + config: coreServices.config, + logger: coreServices.logger, + discovery: coreServices.discovery, + scheduler: coreServices.scheduler, + indexRegistry: searchIndexRegistryExtensionPoint, + }, + async init({ config, logger, discovery, scheduler, indexRegistry }) { + const { schedule, ...rest } = options; + + indexRegistry.addCollator({ + schedule: scheduler.createScheduledTaskRunner(schedule), + factory: ToolDocumentCollatorFactory.fromConfig(config, { + ...rest, + discovery, + logger: loggerToWinstonLogger(logger), + }), + }); + }, + }); + }, + }), +); diff --git a/plugins/explore-backend/src/search/ToolDocumentCollatorFactory.ts b/plugins/search-backend-module-explore/src/collators/ToolDocumentCollatorFactory.ts similarity index 100% rename from plugins/explore-backend/src/search/ToolDocumentCollatorFactory.ts rename to plugins/search-backend-module-explore/src/collators/ToolDocumentCollatorFactory.ts diff --git a/plugins/explore-backend/src/search/index.ts b/plugins/search-backend-module-explore/src/collators/index.ts similarity index 99% rename from plugins/explore-backend/src/search/index.ts rename to plugins/search-backend-module-explore/src/collators/index.ts index 70e9414652..45f0c933e2 100644 --- a/plugins/explore-backend/src/search/index.ts +++ b/plugins/search-backend-module-explore/src/collators/index.ts @@ -15,6 +15,7 @@ */ export { ToolDocumentCollatorFactory } from './ToolDocumentCollatorFactory'; + export type { ToolDocument, ToolDocumentCollatorFactoryOptions, diff --git a/plugins/search-backend-module-explore/src/index.ts b/plugins/search-backend-module-explore/src/index.ts new file mode 100644 index 0000000000..ea94071984 --- /dev/null +++ b/plugins/search-backend-module-explore/src/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @packageDocumentation + * A module for the search backend that exports Explore modules. + */ + +export * from './collators'; diff --git a/plugins/search-backend-module-pg/alpha-api-report.md b/plugins/search-backend-module-pg/alpha-api-report.md index 01a83bace4..eaee78e944 100644 --- a/plugins/search-backend-module-pg/alpha-api-report.md +++ b/plugins/search-backend-module-pg/alpha-api-report.md @@ -6,7 +6,7 @@ import { BackendFeature } from '@backstage/backend-plugin-api'; // @alpha -export const pgSearchEngineModule: () => BackendFeature; +export const searchModulePostgresEngine: () => BackendFeature; // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/search-backend-module-pg/src/alpha.ts b/plugins/search-backend-module-pg/src/alpha.ts index 80ce31adf6..e3e0087957 100644 --- a/plugins/search-backend-module-pg/src/alpha.ts +++ b/plugins/search-backend-module-pg/src/alpha.ts @@ -24,8 +24,8 @@ import { PgSearchEngine } from './PgSearchEngine'; * @alpha * Search backend module for the Postgres engine. */ -export const pgSearchEngineModule = createBackendModule({ - moduleId: 'pgSearchEngineModule', +export const searchModulePostgresEngine = createBackendModule({ + moduleId: 'postgresEngine', pluginId: 'search', register(env) { env.registerInit({ diff --git a/plugins/search-backend-module-techdocs/.eslintrc.js b/plugins/search-backend-module-techdocs/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/search-backend-module-techdocs/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/search-backend-module-techdocs/CHANGELOG.md b/plugins/search-backend-module-techdocs/CHANGELOG.md new file mode 100644 index 0000000000..85d533ae87 --- /dev/null +++ b/plugins/search-backend-module-techdocs/CHANGELOG.md @@ -0,0 +1 @@ +# @backstage/plugin-search-backend-module-techdocs diff --git a/plugins/search-backend-module-techdocs/README.md b/plugins/search-backend-module-techdocs/README.md new file mode 100644 index 0000000000..2711150d56 --- /dev/null +++ b/plugins/search-backend-module-techdocs/README.md @@ -0,0 +1,7 @@ +# search-backend-module-techdocs + +... + +## Getting started + +... diff --git a/plugins/search-backend-module-techdocs/alpha-api-report.md b/plugins/search-backend-module-techdocs/alpha-api-report.md new file mode 100644 index 0000000000..3360055c62 --- /dev/null +++ b/plugins/search-backend-module-techdocs/alpha-api-report.md @@ -0,0 +1,40 @@ +## API Report File for "@backstage/plugin-search-backend-module-techdocs" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +/// + +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { CatalogApi } from '@backstage/catalog-client'; +import { Logger } from 'winston'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { TokenManager } from '@backstage/backend-common'; + +// @alpha +export const searchModuleTechDocsCollator: ( + options: SearchModuleTechDocsCollatorOptions, +) => BackendFeature; + +// @alpha +export type SearchModuleTechDocsCollatorOptions = Omit< + TechDocsCollatorFactoryOptions, + 'logger' | 'discovery' | 'tokenManager' +> & { + schedule: TaskScheduleDefinition; +}; + +// @public +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/search-backend-module-techdocs/api-report.md b/plugins/search-backend-module-techdocs/api-report.md new file mode 100644 index 0000000000..9f72b1c487 --- /dev/null +++ b/plugins/search-backend-module-techdocs/api-report.md @@ -0,0 +1,42 @@ +## API Report File for "@backstage/plugin-search-backend-module-techdocs" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +/// + +import { CatalogApi } from '@backstage/catalog-client'; +import { Config } from '@backstage/config'; +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { Logger } from 'winston'; +import { Permission } from '@backstage/plugin-permission-common'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Readable } from 'stream'; +import { TokenManager } from '@backstage/backend-common'; + +// @public +export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + config: Config, + options: TechDocsCollatorFactoryOptions, + ): DefaultTechDocsCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; +} + +// @public +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; +``` diff --git a/plugins/search-backend-module-techdocs/package.json b/plugins/search-backend-module-techdocs/package.json new file mode 100644 index 0000000000..555e34ebdf --- /dev/null +++ b/plugins/search-backend-module-techdocs/package.json @@ -0,0 +1,71 @@ +{ + "name": "@backstage/plugin-search-backend-module-techdocs", + "description": "A module for the search backend that exports techdocs modules", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } + }, + "backstage": { + "role": "backend-plugin-module" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-client": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", + "@backstage/integration": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", + "@backstage/plugin-permission-common": "workspace:^", + "@backstage/plugin-search-backend-node": "workspace:^", + "@backstage/plugin-search-common": "workspace:^", + "@backstage/plugin-techdocs-node": "workspace:^", + "@types/express": "^4.17.6", + "dockerode": "^3.3.1", + "express": "^4.17.1", + "express-promise-router": "^4.1.0", + "fs-extra": "10.1.0", + "knex": "^2.0.0", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7", + "p-limit": "^3.1.0", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "msw": "^1.0.0" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/search-backend-module-techdocs/src/alpha.ts b/plugins/search-backend-module-techdocs/src/alpha.ts new file mode 100644 index 0000000000..f58c439c80 --- /dev/null +++ b/plugins/search-backend-module-techdocs/src/alpha.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @packageDocumentation + * A module for the search backend that exports TechDocs modules. + */ + +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; + +import { + DefaultTechDocsCollatorFactory, + TechDocsCollatorFactoryOptions, +} from './collators'; + +export type { TechDocsCollatorFactoryOptions }; + +/** + * @alpha + * Options for {@link searchModuleTechDocsCollator}. + */ +export type SearchModuleTechDocsCollatorOptions = Omit< + TechDocsCollatorFactoryOptions, + 'logger' | 'discovery' | 'tokenManager' +> & { + schedule: TaskScheduleDefinition; +}; + +/** + * @alpha + * Search backend module for the TechDocs index. + */ +export const searchModuleTechDocsCollator = createBackendModule( + (options: SearchModuleTechDocsCollatorOptions) => ({ + moduleId: 'techDocsCollator', + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + config: coreServices.config, + logger: coreServices.logger, + discovery: coreServices.discovery, + tokenManager: coreServices.tokenManager, + scheduler: coreServices.scheduler, + indexRegistry: searchIndexRegistryExtensionPoint, + }, + async init({ + config, + logger, + discovery, + tokenManager, + scheduler, + indexRegistry, + }) { + const { schedule, ...rest } = options; + + indexRegistry.addCollator({ + schedule: scheduler.createScheduledTaskRunner(schedule), + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { + ...rest, + discovery, + tokenManager, + logger: loggerToWinstonLogger(logger), + }), + }); + }, + }); + }, + }), +); diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts similarity index 100% rename from plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts rename to plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts similarity index 100% rename from plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts rename to plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts diff --git a/plugins/search-backend-module-techdocs/src/collators/index.ts b/plugins/search-backend-module-techdocs/src/collators/index.ts new file mode 100644 index 0000000000..8c5fd122f1 --- /dev/null +++ b/plugins/search-backend-module-techdocs/src/collators/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; + +export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory'; diff --git a/plugins/search-backend-module-techdocs/src/index.ts b/plugins/search-backend-module-techdocs/src/index.ts new file mode 100644 index 0000000000..c28175fd61 --- /dev/null +++ b/plugins/search-backend-module-techdocs/src/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @packageDocumentation + * A module for the search backend that exports TechDocs modules. + */ + +export * from './collators'; diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 1f6aa1d66a..2abd9783f7 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -3,12 +3,10 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -/// - import { CatalogApi } from '@backstage/catalog-client'; import { CatalogClient } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-search-backend-module-techdocs'; import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { GeneratorBuilder } from '@backstage/plugin-techdocs-node'; @@ -19,7 +17,7 @@ import { PluginCacheManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PreparerBuilder } from '@backstage/plugin-techdocs-node'; import { PublisherBase } from '@backstage/plugin-techdocs-node'; -import { Readable } from 'stream'; +import { TechDocsCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-techdocs'; import { TechDocsDocument } from '@backstage/plugin-techdocs-node'; import { TokenManager } from '@backstage/backend-common'; import * as winston from 'winston'; @@ -47,20 +45,7 @@ export class DefaultTechDocsCollator { readonly visibilityPermission: Permission; } -// @public -export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { - // (undocumented) - static fromConfig( - config: Config, - options: TechDocsCollatorFactoryOptions, - ): DefaultTechDocsCollatorFactory; - // (undocumented) - getCollator(): Promise; - // (undocumented) - readonly type: string; - // (undocumented) - readonly visibilityPermission: Permission; -} +export { DefaultTechDocsCollatorFactory }; // @public export interface DocsBuildStrategy { @@ -105,16 +90,7 @@ export type ShouldBuildParameters = { entity: Entity; }; -// @public -export type TechDocsCollatorFactoryOptions = { - discovery: PluginEndpointDiscovery; - logger: Logger; - tokenManager: TokenManager; - locationTemplate?: string; - catalogClient?: CatalogApi; - parallelismLimit?: number; - legacyPathCasing?: boolean; -}; +export { TechDocsCollatorFactoryOptions }; // @public export type TechDocsCollatorOptions = { diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 1d6ca8931e..a9f7a5c9b2 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -57,6 +57,7 @@ "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", + "@backstage/plugin-search-backend-module-techdocs": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/plugin-techdocs-node": "workspace:^", "@types/express": "^4.17.6", diff --git a/plugins/techdocs-backend/src/search/index.ts b/plugins/techdocs-backend/src/search/index.ts index 68e3b4edc7..0c403ef0d7 100644 --- a/plugins/techdocs-backend/src/search/index.ts +++ b/plugins/techdocs-backend/src/search/index.ts @@ -14,11 +14,20 @@ * limitations under the License. */ -export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; -export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory'; - /** * todo(backstage/techdocs-core): stop exporting these in a future release. */ export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator'; + +/** + * @deprecated + * import from @backstage/search-backend-module-techdocs instead + */ +export type { TechDocsCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-techdocs'; + +/** + * @deprecated + * import from @backstage/search-backend-module-techdocs instead + */ +export { DefaultTechDocsCollatorFactory } from '@backstage/plugin-search-backend-module-techdocs'; diff --git a/yarn.lock b/yarn.lock index b5203fbe29..2046cd9ece 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5207,6 +5207,7 @@ __metadata: "@backstage/plugin-permission-common": "workspace:^" "@backstage/plugin-permission-node": "workspace:^" "@backstage/plugin-scaffolder-common": "workspace:^" + "@backstage/plugin-search-backend-module-catalog": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/types": "workspace:^" @@ -6048,6 +6049,7 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/plugin-explore-common": "workspace:^" + "@backstage/plugin-search-backend-module-explore": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@types/express": "*" "@types/supertest": ^2.0.8 @@ -7768,6 +7770,37 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-search-backend-module-catalog@workspace:^, @backstage/plugin-search-backend-module-catalog@workspace:plugins/search-backend-module-catalog": + version: 0.0.0-use.local + resolution: "@backstage/plugin-search-backend-module-catalog@workspace:plugins/search-backend-module-catalog" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" + "@backstage/catalog-client": "workspace:^" + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" + "@backstage/plugin-permission-common": "workspace:^" + "@backstage/plugin-search-backend-node": "workspace:^" + "@backstage/plugin-search-common": "workspace:^" + "@elastic/elasticsearch": ^7.13.0 + "@elastic/elasticsearch-mock": ^1.0.0 + "@opensearch-project/opensearch": ^2.0.0 + "@short.io/opensearch-mock": ^0.3.1 + aws-os-connection: ^0.2.0 + aws-sdk: ^2.948.0 + elastic-builder: ^2.16.0 + lodash: ^4.17.21 + msw: ^1.0.0 + uuid: ^8.3.2 + winston: ^3.2.1 + languageName: unknown + linkType: soft + "@backstage/plugin-search-backend-module-elasticsearch@workspace:^, @backstage/plugin-search-backend-module-elasticsearch@workspace:plugins/search-backend-module-elasticsearch": version: 0.0.0-use.local resolution: "@backstage/plugin-search-backend-module-elasticsearch@workspace:plugins/search-backend-module-elasticsearch" @@ -7791,6 +7824,39 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-search-backend-module-explore@workspace:^, @backstage/plugin-search-backend-module-explore@workspace:plugins/search-backend-module-explore": + version: 0.0.0-use.local + resolution: "@backstage/plugin-search-backend-module-explore@workspace:plugins/search-backend-module-explore" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" + "@backstage/catalog-client": "workspace:^" + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/integration": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" + "@backstage/plugin-explore-common": "workspace:^" + "@backstage/plugin-permission-common": "workspace:^" + "@backstage/plugin-search-backend-node": "workspace:^" + "@backstage/plugin-search-common": "workspace:^" + "@types/express": ^4.17.6 + dockerode: ^3.3.1 + express: ^4.17.1 + express-promise-router: ^4.1.0 + fs-extra: 10.1.0 + knex: ^2.0.0 + lodash: ^4.17.21 + msw: ^1.0.0 + node-fetch: ^2.6.7 + p-limit: ^3.1.0 + winston: ^3.2.1 + languageName: unknown + linkType: soft + "@backstage/plugin-search-backend-module-pg@workspace:^, @backstage/plugin-search-backend-module-pg@workspace:plugins/search-backend-module-pg": version: 0.0.0-use.local resolution: "@backstage/plugin-search-backend-module-pg@workspace:plugins/search-backend-module-pg" @@ -7809,6 +7875,39 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-search-backend-module-techdocs@workspace:^, @backstage/plugin-search-backend-module-techdocs@workspace:plugins/search-backend-module-techdocs": + version: 0.0.0-use.local + resolution: "@backstage/plugin-search-backend-module-techdocs@workspace:plugins/search-backend-module-techdocs" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" + "@backstage/catalog-client": "workspace:^" + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/integration": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" + "@backstage/plugin-permission-common": "workspace:^" + "@backstage/plugin-search-backend-node": "workspace:^" + "@backstage/plugin-search-common": "workspace:^" + "@backstage/plugin-techdocs-node": "workspace:^" + "@types/express": ^4.17.6 + dockerode: ^3.3.1 + express: ^4.17.1 + express-promise-router: ^4.1.0 + fs-extra: 10.1.0 + knex: ^2.0.0 + lodash: ^4.17.21 + msw: ^1.0.0 + node-fetch: ^2.6.7 + p-limit: ^3.1.0 + winston: ^3.2.1 + languageName: unknown + linkType: soft + "@backstage/plugin-search-backend-node@workspace:^, @backstage/plugin-search-backend-node@workspace:plugins/search-backend-node": version: 0.0.0-use.local resolution: "@backstage/plugin-search-backend-node@workspace:plugins/search-backend-node" @@ -8371,6 +8470,7 @@ __metadata: "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" + "@backstage/plugin-search-backend-module-techdocs": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/plugin-techdocs-node": "workspace:^" @@ -22490,11 +22590,12 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/plugin-app-backend": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" - "@backstage/plugin-explore-backend": "workspace:^" "@backstage/plugin-scaffolder-backend": "workspace:^" "@backstage/plugin-search-backend": "workspace:^" + "@backstage/plugin-search-backend-module-catalog": "workspace:^" + "@backstage/plugin-search-backend-module-explore": "workspace:^" + "@backstage/plugin-search-backend-module-techdocs": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" - "@backstage/plugin-techdocs-backend": "workspace:^" "@backstage/plugin-todo-backend": "workspace:^" languageName: unknown linkType: soft From d4c66877b07b93e630b30ad2a6d4c7487e25186c Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Tue, 7 Mar 2023 15:12:19 +0100 Subject: [PATCH 17/27] feat(search): add tests for modules Co-authored-by: Emma Indal Signed-off-by: Camila Belo --- .../src/alpha.test.ts | 46 +++++++++++++++++++ .../src/alpha.test.ts | 46 +++++++++++++++++++ .../src/alpha.test.ts | 46 +++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 plugins/search-backend-module-catalog/src/alpha.test.ts create mode 100644 plugins/search-backend-module-explore/src/alpha.test.ts create mode 100644 plugins/search-backend-module-techdocs/src/alpha.test.ts diff --git a/plugins/search-backend-module-catalog/src/alpha.test.ts b/plugins/search-backend-module-catalog/src/alpha.test.ts new file mode 100644 index 0000000000..8a3082c6aa --- /dev/null +++ b/plugins/search-backend-module-catalog/src/alpha.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { startTestBackend } from '@backstage/backend-test-utils'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; +import { searchModuleCatalogCollator } from './alpha'; + +describe('searchModuleCatalogCollator', () => { + const schedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, + }; + + it('should register the catalog collator to the search index registry extension point with factory and schedule', async () => { + const extensionPointMock = { + addCollator: jest.fn(), + }; + + await startTestBackend({ + extensionPoints: [ + [searchIndexRegistryExtensionPoint, extensionPointMock], + ], + features: [searchModuleCatalogCollator({ schedule })], + }); + + expect(extensionPointMock.addCollator).toHaveBeenCalledTimes(1); + expect(extensionPointMock.addCollator).toHaveBeenCalledWith({ + factory: expect.objectContaining({ type: 'software-catalog' }), + schedule: expect.objectContaining({ run: expect.any(Function) }), + }); + }); +}); diff --git a/plugins/search-backend-module-explore/src/alpha.test.ts b/plugins/search-backend-module-explore/src/alpha.test.ts new file mode 100644 index 0000000000..9b133f0fd5 --- /dev/null +++ b/plugins/search-backend-module-explore/src/alpha.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { startTestBackend } from '@backstage/backend-test-utils'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; +import { searchModuleExploreCollator } from './alpha'; + +describe('searchModuleExploreCollator', () => { + const schedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, + }; + + it('should register the explore collator to the search index registry extension point with factory and schedule', async () => { + const extensionPointMock = { + addCollator: jest.fn(), + }; + + await startTestBackend({ + extensionPoints: [ + [searchIndexRegistryExtensionPoint, extensionPointMock], + ], + features: [searchModuleExploreCollator({ schedule })], + }); + + expect(extensionPointMock.addCollator).toHaveBeenCalledTimes(1); + expect(extensionPointMock.addCollator).toHaveBeenCalledWith({ + factory: expect.objectContaining({ type: 'tools' }), + schedule: expect.objectContaining({ run: expect.any(Function) }), + }); + }); +}); diff --git a/plugins/search-backend-module-techdocs/src/alpha.test.ts b/plugins/search-backend-module-techdocs/src/alpha.test.ts new file mode 100644 index 0000000000..011b00b12c --- /dev/null +++ b/plugins/search-backend-module-techdocs/src/alpha.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { startTestBackend } from '@backstage/backend-test-utils'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; +import { searchModuleTechDocsCollator } from './alpha'; + +describe('searchModuleTechDocsCollator', () => { + const schedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, + }; + + it('should register the techdocs collator to the search index registry extension point with factory and schedule', async () => { + const extensionPointMock = { + addCollator: jest.fn(), + }; + + await startTestBackend({ + extensionPoints: [ + [searchIndexRegistryExtensionPoint, extensionPointMock], + ], + features: [searchModuleTechDocsCollator({ schedule })], + }); + + expect(extensionPointMock.addCollator).toHaveBeenCalledTimes(1); + expect(extensionPointMock.addCollator).toHaveBeenCalledWith({ + factory: expect.objectContaining({ type: 'techdocs' }), + schedule: expect.objectContaining({ run: expect.any(Function) }), + }); + }); +}); From 01ae205352e1890bb32dd32bfc67d3e84024fde2 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Tue, 7 Mar 2023 15:23:28 +0100 Subject: [PATCH 18/27] feat(search): update docs Co-authored-by: Emma Indal Signed-off-by: Camila Belo --- .changeset/nine-clouds-flow.md | 5 -- .changeset/tall-chairs-explain.md | 7 ++ .changeset/wet-lamps-happen.md | 7 ++ docs/features/search/how-to-guides.md | 86 ++++--------------- .../integrating-search-into-plugins.md | 2 +- .../CHANGELOG.md | 1 - .../search-backend-module-catalog/README.md | 24 +++++- .../CHANGELOG.md | 1 - .../search-backend-module-explore/README.md | 24 +++++- .../package.json | 1 - .../CHANGELOG.md | 1 - .../search-backend-module-techdocs/README.md | 24 +++++- .../package.json | 1 - yarn.lock | 3 +- 14 files changed, 101 insertions(+), 86 deletions(-) delete mode 100644 .changeset/nine-clouds-flow.md create mode 100644 .changeset/tall-chairs-explain.md create mode 100644 .changeset/wet-lamps-happen.md delete mode 100644 plugins/search-backend-module-catalog/CHANGELOG.md delete mode 100644 plugins/search-backend-module-explore/CHANGELOG.md delete mode 100644 plugins/search-backend-module-techdocs/CHANGELOG.md diff --git a/.changeset/nine-clouds-flow.md b/.changeset/nine-clouds-flow.md deleted file mode 100644 index fca6bbbf1c..0000000000 --- a/.changeset/nine-clouds-flow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'example-backend-next': patch ---- - -Adds search plugin and search index registry to backend. diff --git a/.changeset/tall-chairs-explain.md b/.changeset/tall-chairs-explain.md new file mode 100644 index 0000000000..e7367d2d4a --- /dev/null +++ b/.changeset/tall-chairs-explain.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-techdocs-backend': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-explore-backend': patch +--- + +Deprecate the collator files in the backend package by extracting them into a separate module. diff --git a/.changeset/wet-lamps-happen.md b/.changeset/wet-lamps-happen.md new file mode 100644 index 0000000000..76e36d219d --- /dev/null +++ b/.changeset/wet-lamps-happen.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-search-backend-module-techdocs': minor +'@backstage/plugin-search-backend-module-catalog': minor +'@backstage/plugin-search-backend-module-explore': minor +--- + +Extend the index with custom collators by creating search plugin modules. diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index e8f58e9f26..da94bd4ca4 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -51,7 +51,7 @@ The TechDocs plugin has supported integrations to Search, meaning that it provides a default collator factory ready to be used. The purpose of this guide is to walk you through how to register the -[DefaultTechDocsCollatorFactory](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts) +[DefaultTechDocsCollatorFactory](https://github.com/backstage/backstage/blob/de294ce5c410c9eb56da6870a1fab795268f60e3/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts) in your App, so that you can get TechDocs documents indexed. If you have been through the @@ -376,78 +376,30 @@ There are other more specific search results layout components that also accept Recently, the Backstage maintainers [announced the new Backend System](https://backstage.io/blog/2023/02/15/backend-system-alpha). The search plugins are now migrated to support the new backend system. In this guide you will learn how to update your backend set up. -1. In packages/backend/search.ts - -```ts -import { - coreServices, - createBackendModule, -} from '@backstage/backend-plugin-api'; -import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; -import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; -import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; -import { ToolDocumentCollatorFactory } from '@backstage/plugin-explore-backend'; -import { loggerToWinstonLogger } from '@backstage/backend-common'; - -export const searchIndexRegistry = createBackendModule({ - moduleId: 'searchIndexRegistry', - pluginId: 'search', - register(env) { - env.registerInit({ - deps: { - indexRegistry: searchIndexRegistryExtensionPoint, - config: coreServices.config, - discovery: coreServices.discovery, - tokenManager: coreServices.tokenManager, - logger: coreServices.logger, - scheduler: coreServices.scheduler, - }, - async init({ - indexRegistry, - config, - logger, - discovery, - tokenManager, - scheduler, - }) { - // define scheule, the same way you did it before - const schedule = scheduler.createScheduledTaskRunner({ - frequency: { minutes: 10 }, - timeout: { minutes: 15 }, - // A 3 second delay gives the backend server a chance to initialize before - // any collators are executed, which may attempt requests against the API. - initialDelay: { seconds: 3 }, - }); - - indexRegistry.addCollator({ - schedule, - factory: DefaultCatalogCollatorFactory.fromConfig(config, { - discovery, - tokenManager, - }), - }); - - // .... other collators and decorators - }, - }); - }, -}); -``` - -2. In packages/backend/index.ts +In packages/backend-next/index.ts ```ts import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; -import { elasticSearchEngineModule } from '@backstage/plugin-search-backend-module-elasticsearch/alpha'; -import { searchIndexRegistry } from './plugins/search'; +import { searchModuleElasticsearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch/alpha'; +import { searchModuleCatalogCollator } from '@backstage/plugin-search-backend-module-catalog/alpha'; +import { searchModuleTechDocsCollator } from '@backstage/plugin-search-backend-module-techdocs/alpha'; +import { searchModuleExploreCollator } from '@backstage/plugin-search-backend-module-explore/alpha'; + +const schedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, +}; const backend = createBackend(); -// ...other modules +// adding the search plugin to the backend backend.add(searchPlugin()); -backend.add(searchIndexRegistry()); - -// the default search engine is Lunr, if you want to extend the search backend with another search engine. -backend.add(elasticSearchEngineModule()); +// (optional) the default search engine is Lunr, if you want to extend the search backend with another search engine. +backend.add(searchModuleElasticsearchEngine()); +// extending search with collator modules to start index documents +backend.add(searchModuleCatalogCollator({ schedule })); +backend.add(searchModuleTechDocsCollator({ schedule })); +backend.add(searchModuleExploreCollator({ schedule })); backend.start(); ``` diff --git a/docs/plugins/integrating-search-into-plugins.md b/docs/plugins/integrating-search-into-plugins.md index 4e829b6ae9..7fb8e40b20 100644 --- a/docs/plugins/integrating-search-into-plugins.md +++ b/docs/plugins/integrating-search-into-plugins.md @@ -142,7 +142,7 @@ export class FaqCollatorFactory implements DocumentCollatorFactory { To verify your implementation works as expected make sure to add tests for it. For your convenience, there is the [`TestPipeline`](https://backstage.io/docs/reference/plugin-search-backend-node.testpipeline) utility that emulates a pipeline into which you can integrate your custom collator. -Look at [DefaultTechDocsCollatorFactory test](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts), for an example. +Look at [DefaultTechDocsCollatorFactory test](https://github.com/backstage/backstage/blob/de294ce5c410c9eb56da6870a1fab795268f60e3/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts), for an example. #### 6. Make your plugins collator discoverable for others diff --git a/plugins/search-backend-module-catalog/CHANGELOG.md b/plugins/search-backend-module-catalog/CHANGELOG.md deleted file mode 100644 index c870819424..0000000000 --- a/plugins/search-backend-module-catalog/CHANGELOG.md +++ /dev/null @@ -1 +0,0 @@ -# @backstage/plugin-search-backend-module-catalog diff --git a/plugins/search-backend-module-catalog/README.md b/plugins/search-backend-module-catalog/README.md index 9860f86b1d..b6c734d893 100644 --- a/plugins/search-backend-module-catalog/README.md +++ b/plugins/search-backend-module-catalog/README.md @@ -1,7 +1,27 @@ # search-backend-module-catalog -... +> DISCLAIMER: The new backend system is in alpha, and so are the search backend module support for the new backend system. We don't recommend you to migrate your backend installations to the new system yet. But if you want to experiment, you can find getting started guides below. -## Getting started +This package exports catalog backend modules responsible for extending search. + +## Example + +```tsx +// packages/backend-next/src/index.ts +import { createBackend } from '@backstage/backend-defaults'; +import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; +import { searchModuleCatalogCollator } from '@backstage/plugin-search-backend-module-catalog/alpha'; + +const schedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, +}; + +const backend = createBackend(); +backend.add(searchPlugin()); +backend.add(searchModuleCatalogCollator({ schedule })); +backend.start(); +``` ... diff --git a/plugins/search-backend-module-explore/CHANGELOG.md b/plugins/search-backend-module-explore/CHANGELOG.md deleted file mode 100644 index 8640d14b73..0000000000 --- a/plugins/search-backend-module-explore/CHANGELOG.md +++ /dev/null @@ -1 +0,0 @@ -# @backstage/plugin-search-backend-module-explore diff --git a/plugins/search-backend-module-explore/README.md b/plugins/search-backend-module-explore/README.md index bbc8392f14..d2b00767e2 100644 --- a/plugins/search-backend-module-explore/README.md +++ b/plugins/search-backend-module-explore/README.md @@ -1,7 +1,27 @@ # search-backend-module-explore -... +> DISCLAIMER: The new backend system is in alpha, and so are the search backend module support for the new backend system. We don't recommend you to migrate your backend installations to the new system yet. But if you want to experiment, you can find getting started guides below. -## Getting started +This package exports explore backend modules responsible for extending search. + +## Example + +```tsx +// packages/backend-next/src/index.ts +import { createBackend } from '@backstage/backend-defaults'; +import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; +import { searchModuleExploreCollator } from '@backstage/plugin-search-backend-module-explore/alpha'; + +const schedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, +}; + +const backend = createBackend(); +backend.add(searchPlugin()); +backend.add(searchModuleExploreCollator({ schedule })); +backend.start(); +``` ... diff --git a/plugins/search-backend-module-explore/package.json b/plugins/search-backend-module-explore/package.json index 356059f845..c943f4480d 100644 --- a/plugins/search-backend-module-explore/package.json +++ b/plugins/search-backend-module-explore/package.json @@ -49,7 +49,6 @@ "@backstage/plugin-permission-common": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", - "@types/express": "^4.17.6", "dockerode": "^3.3.1", "express": "^4.17.1", "express-promise-router": "^4.1.0", diff --git a/plugins/search-backend-module-techdocs/CHANGELOG.md b/plugins/search-backend-module-techdocs/CHANGELOG.md deleted file mode 100644 index 85d533ae87..0000000000 --- a/plugins/search-backend-module-techdocs/CHANGELOG.md +++ /dev/null @@ -1 +0,0 @@ -# @backstage/plugin-search-backend-module-techdocs diff --git a/plugins/search-backend-module-techdocs/README.md b/plugins/search-backend-module-techdocs/README.md index 2711150d56..817bb40998 100644 --- a/plugins/search-backend-module-techdocs/README.md +++ b/plugins/search-backend-module-techdocs/README.md @@ -1,7 +1,27 @@ # search-backend-module-techdocs -... +> DISCLAIMER: The new backend system is in alpha, and so are the search backend module support for the new backend system. We don't recommend you to migrate your backend installations to the new system yet. But if you want to experiment, you can find getting started guides below. -## Getting started +This package exports techdocs backend modules responsible for extending search. + +## Example + +```tsx +// packages/backend-next/src/index.ts +import { createBackend } from '@backstage/backend-defaults'; +import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; +import { searchModuleTechDocsCollator } from '@backstage/plugin-search-backend-module-techdocs/alpha'; + +const schedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, +}; + +const backend = createBackend(); +backend.add(searchPlugin()); +backend.add(searchModuleTechDocsCollator({ schedule })); +backend.start(); +``` ... diff --git a/plugins/search-backend-module-techdocs/package.json b/plugins/search-backend-module-techdocs/package.json index 555e34ebdf..0e7a350938 100644 --- a/plugins/search-backend-module-techdocs/package.json +++ b/plugins/search-backend-module-techdocs/package.json @@ -49,7 +49,6 @@ "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/plugin-techdocs-node": "workspace:^", - "@types/express": "^4.17.6", "dockerode": "^3.3.1", "express": "^4.17.1", "express-promise-router": "^4.1.0", diff --git a/yarn.lock b/yarn.lock index 2046cd9ece..c4d07145e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7843,7 +7843,6 @@ __metadata: "@backstage/plugin-permission-common": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" - "@types/express": ^4.17.6 dockerode: ^3.3.1 express: ^4.17.1 express-promise-router: ^4.1.0 @@ -7894,7 +7893,6 @@ __metadata: "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/plugin-techdocs-node": "workspace:^" - "@types/express": ^4.17.6 dockerode: ^3.3.1 express: ^4.17.1 express-promise-router: ^4.1.0 @@ -22596,6 +22594,7 @@ __metadata: "@backstage/plugin-search-backend-module-explore": "workspace:^" "@backstage/plugin-search-backend-module-techdocs": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" + "@backstage/plugin-techdocs-backend": "workspace:^" "@backstage/plugin-todo-backend": "workspace:^" languageName: unknown linkType: soft From 5e2ccf01831d5f8db69d3e6fcc3e750b0b88ee58 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Thu, 9 Mar 2023 08:28:18 +0100 Subject: [PATCH 19/27] use correct version of new package Co-authored-by: Patrik Oldsberg Signed-off-by: Emma Indal --- plugins/search-backend-module-catalog/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-backend-module-catalog/package.json b/plugins/search-backend-module-catalog/package.json index 6c7b78437c..011e4d93df 100644 --- a/plugins/search-backend-module-catalog/package.json +++ b/plugins/search-backend-module-catalog/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/plugin-search-backend-module-catalog", "description": "A module for the search backend that exports catalog modules", - "version": "1.1.4-next.0", + "version": "0.0.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", From 3dc6a4a0811c2d4b368fadd22cc99c2d8a8ee2b7 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Thu, 9 Mar 2023 13:37:31 +0100 Subject: [PATCH 20/27] change postgres module to patch in changeset Co-authored-by: Patrik Oldsberg Signed-off-by: Emma Indal --- .changeset/giant-hairs-serve.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/giant-hairs-serve.md b/.changeset/giant-hairs-serve.md index ae450234d1..0f77a6fe34 100644 --- a/.changeset/giant-hairs-serve.md +++ b/.changeset/giant-hairs-serve.md @@ -1,6 +1,6 @@ --- '@backstage/plugin-search-backend-module-elasticsearch': minor -'@backstage/plugin-search-backend-module-pg': minor +'@backstage/plugin-search-backend-module-pg': patch --- Search backend modules migrated to the new backend system. For documentation on how to migrate, check out the [how to migrate to the new backend system guide](../docs/features/search/how-to-guides.md#how-to-migrate-to-use-search-together-with-the-new-backend-system). From 6da5eb1e50f7068c0f4ad38994d88c5913c3e055 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Thu, 9 Mar 2023 13:53:11 +0100 Subject: [PATCH 21/27] use default schedule for collator modules and make schedule option opotional Signed-off-by: Emma Indal --- docs/features/search/how-to-guides.md | 14 ++++---------- packages/backend-next/src/index.ts | 11 +++-------- .../search-backend-module-catalog/src/alpha.ts | 16 +++++++++++----- .../search-backend-module-explore/src/alpha.ts | 16 +++++++++++----- .../search-backend-module-techdocs/src/alpha.ts | 16 +++++++++++----- 5 files changed, 40 insertions(+), 33 deletions(-) diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index da94bd4ca4..161ff6844c 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -385,21 +385,15 @@ import { searchModuleCatalogCollator } from '@backstage/plugin-search-backend-mo import { searchModuleTechDocsCollator } from '@backstage/plugin-search-backend-module-techdocs/alpha'; import { searchModuleExploreCollator } from '@backstage/plugin-search-backend-module-explore/alpha'; -const schedule = { - frequency: { minutes: 10 }, - timeout: { minutes: 15 }, - initialDelay: { seconds: 3 }, -}; - const backend = createBackend(); // adding the search plugin to the backend backend.add(searchPlugin()); // (optional) the default search engine is Lunr, if you want to extend the search backend with another search engine. backend.add(searchModuleElasticsearchEngine()); -// extending search with collator modules to start index documents -backend.add(searchModuleCatalogCollator({ schedule })); -backend.add(searchModuleTechDocsCollator({ schedule })); -backend.add(searchModuleExploreCollator({ schedule })); +// extending search with collator modules to start index documents, take in optional schedule parameters. +backend.add(searchModuleCatalogCollator()); +backend.add(searchModuleTechDocsCollator()); +backend.add(searchModuleExploreCollator()); backend.start(); ``` diff --git a/packages/backend-next/src/index.ts b/packages/backend-next/src/index.ts index 231343203a..e3cbbd53b8 100644 --- a/packages/backend-next/src/index.ts +++ b/packages/backend-next/src/index.ts @@ -40,14 +40,9 @@ backend.add(catalogPlugin()); backend.add(catalogModuleTemplateKind()); // Search -const schedule = { - frequency: { minutes: 10 }, - timeout: { minutes: 15 }, - initialDelay: { seconds: 3 }, -}; backend.add(searchPlugin()); -backend.add(searchModuleCatalogCollator({ schedule })); -backend.add(searchModuleTechDocsCollator({ schedule })); -backend.add(searchModuleExploreCollator({ schedule })); +backend.add(searchModuleCatalogCollator()); +backend.add(searchModuleTechDocsCollator()); +backend.add(searchModuleExploreCollator()); backend.start(); diff --git a/plugins/search-backend-module-catalog/src/alpha.ts b/plugins/search-backend-module-catalog/src/alpha.ts index d4530b582b..89c7479035 100644 --- a/plugins/search-backend-module-catalog/src/alpha.ts +++ b/plugins/search-backend-module-catalog/src/alpha.ts @@ -46,7 +46,7 @@ export type SearchModuleCatalogCollatorOptions = Omit< DefaultCatalogCollatorFactoryOptions, 'discovery' | 'tokenManager' > & { - schedule: TaskScheduleDefinition; + schedule?: TaskScheduleDefinition; }; /** @@ -54,7 +54,7 @@ export type SearchModuleCatalogCollatorOptions = Omit< * Search backend module for the Catalog index. */ export const searchModuleCatalogCollator = createBackendModule( - (options: SearchModuleCatalogCollatorOptions) => ({ + (options?: SearchModuleCatalogCollatorOptions) => ({ moduleId: 'catalogCollator', pluginId: 'search', register(env) { @@ -73,12 +73,18 @@ export const searchModuleCatalogCollator = createBackendModule( scheduler, indexRegistry, }) { - const { schedule, ...rest } = options; + const defaultSchedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, + }; indexRegistry.addCollator({ - schedule: scheduler.createScheduledTaskRunner(schedule), + schedule: scheduler.createScheduledTaskRunner( + options?.schedule ?? defaultSchedule, + ), factory: DefaultCatalogCollatorFactory.fromConfig(config, { - ...rest, + ...options, discovery, tokenManager, }), diff --git a/plugins/search-backend-module-explore/src/alpha.ts b/plugins/search-backend-module-explore/src/alpha.ts index 357bb4233e..01303db631 100644 --- a/plugins/search-backend-module-explore/src/alpha.ts +++ b/plugins/search-backend-module-explore/src/alpha.ts @@ -47,7 +47,7 @@ export type SearchModuleExploreCollatorOptions = Omit< ToolDocumentCollatorFactoryOptions, 'logger' | 'discovery' > & { - schedule: TaskScheduleDefinition; + schedule?: TaskScheduleDefinition; }; /** @@ -55,7 +55,7 @@ export type SearchModuleExploreCollatorOptions = Omit< * Search backend module for the Explore index. */ export const searchModuleExploreCollator = createBackendModule( - (options: SearchModuleExploreCollatorOptions) => ({ + (options?: SearchModuleExploreCollatorOptions) => ({ moduleId: 'exploreCollator', pluginId: 'search', register(env) { @@ -68,12 +68,18 @@ export const searchModuleExploreCollator = createBackendModule( indexRegistry: searchIndexRegistryExtensionPoint, }, async init({ config, logger, discovery, scheduler, indexRegistry }) { - const { schedule, ...rest } = options; + const defaultSchedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, + }; indexRegistry.addCollator({ - schedule: scheduler.createScheduledTaskRunner(schedule), + schedule: scheduler.createScheduledTaskRunner( + options?.schedule ?? defaultSchedule, + ), factory: ToolDocumentCollatorFactory.fromConfig(config, { - ...rest, + ...options, discovery, logger: loggerToWinstonLogger(logger), }), diff --git a/plugins/search-backend-module-techdocs/src/alpha.ts b/plugins/search-backend-module-techdocs/src/alpha.ts index f58c439c80..c781ad40ca 100644 --- a/plugins/search-backend-module-techdocs/src/alpha.ts +++ b/plugins/search-backend-module-techdocs/src/alpha.ts @@ -42,7 +42,7 @@ export type SearchModuleTechDocsCollatorOptions = Omit< TechDocsCollatorFactoryOptions, 'logger' | 'discovery' | 'tokenManager' > & { - schedule: TaskScheduleDefinition; + schedule?: TaskScheduleDefinition; }; /** @@ -50,7 +50,7 @@ export type SearchModuleTechDocsCollatorOptions = Omit< * Search backend module for the TechDocs index. */ export const searchModuleTechDocsCollator = createBackendModule( - (options: SearchModuleTechDocsCollatorOptions) => ({ + (options?: SearchModuleTechDocsCollatorOptions) => ({ moduleId: 'techDocsCollator', pluginId: 'search', register(env) { @@ -71,12 +71,18 @@ export const searchModuleTechDocsCollator = createBackendModule( scheduler, indexRegistry, }) { - const { schedule, ...rest } = options; + const defaultSchedule = { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, + }; indexRegistry.addCollator({ - schedule: scheduler.createScheduledTaskRunner(schedule), + schedule: scheduler.createScheduledTaskRunner( + options?.schedule ?? defaultSchedule, + ), factory: DefaultTechDocsCollatorFactory.fromConfig(config, { - ...rest, + ...options, discovery, tokenManager, logger: loggerToWinstonLogger(logger), From b8267644d846c8082a705efd3aabe71e4700c3c8 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Thu, 9 Mar 2023 14:24:03 +0100 Subject: [PATCH 22/27] clean up unused dependencies Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- packages/backend-next/package.json | 2 -- .../package.json | 13 +------ .../package.json | 16 +-------- .../package.json | 7 ---- yarn.lock | 34 ------------------- 5 files changed, 2 insertions(+), 70 deletions(-) diff --git a/packages/backend-next/package.json b/packages/backend-next/package.json index 9859717fd5..246eb348d4 100644 --- a/packages/backend-next/package.json +++ b/packages/backend-next/package.json @@ -25,9 +25,7 @@ "clean": "backstage-cli package clean" }, "dependencies": { - "@backstage/backend-common": "workspace:^", "@backstage/backend-defaults": "workspace:^", - "@backstage/backend-plugin-api": "workspace:^", "@backstage/plugin-app-backend": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-scaffolder-backend": "workspace:^", diff --git a/plugins/search-backend-module-catalog/package.json b/plugins/search-backend-module-catalog/package.json index 011e4d93df..4c323ae7fd 100644 --- a/plugins/search-backend-module-catalog/package.json +++ b/plugins/search-backend-module-catalog/package.json @@ -42,26 +42,15 @@ "@backstage/catalog-client": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", - "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", - "@backstage/plugin-search-common": "workspace:^", - "@elastic/elasticsearch": "^7.13.0", - "@opensearch-project/opensearch": "^2.0.0", - "aws-os-connection": "^0.2.0", - "aws-sdk": "^2.948.0", - "elastic-builder": "^2.16.0", - "lodash": "^4.17.21", - "uuid": "^8.3.2", - "winston": "^3.2.1" + "@backstage/plugin-search-common": "workspace:^" }, "devDependencies": { "@backstage/backend-common": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", - "@elastic/elasticsearch-mock": "^1.0.0", - "@short.io/opensearch-mock": "^0.3.1", "msw": "^1.0.0" }, "files": [ diff --git a/plugins/search-backend-module-explore/package.json b/plugins/search-backend-module-explore/package.json index c943f4480d..e4c71448ff 100644 --- a/plugins/search-backend-module-explore/package.json +++ b/plugins/search-backend-module-explore/package.json @@ -39,30 +39,16 @@ "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", - "@backstage/catalog-client": "workspace:^", - "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", - "@backstage/errors": "workspace:^", - "@backstage/integration": "workspace:^", - "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-explore-common": "workspace:^", - "@backstage/plugin-permission-common": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", - "dockerode": "^3.3.1", - "express": "^4.17.1", - "express-promise-router": "^4.1.0", - "fs-extra": "10.1.0", - "knex": "^2.0.0", - "lodash": "^4.17.21", "node-fetch": "^2.6.7", - "p-limit": "^3.1.0", "winston": "^3.2.1" }, "devDependencies": { "@backstage/backend-test-utils": "workspace:^", - "@backstage/cli": "workspace:^", - "msw": "^1.0.0" + "@backstage/cli": "workspace:^" }, "files": [ "dist" diff --git a/plugins/search-backend-module-techdocs/package.json b/plugins/search-backend-module-techdocs/package.json index 0e7a350938..d08193be29 100644 --- a/plugins/search-backend-module-techdocs/package.json +++ b/plugins/search-backend-module-techdocs/package.json @@ -42,18 +42,11 @@ "@backstage/catalog-client": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", - "@backstage/errors": "workspace:^", - "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/plugin-techdocs-node": "workspace:^", - "dockerode": "^3.3.1", - "express": "^4.17.1", - "express-promise-router": "^4.1.0", - "fs-extra": "10.1.0", - "knex": "^2.0.0", "lodash": "^4.17.21", "node-fetch": "^2.6.7", "p-limit": "^3.1.0", diff --git a/yarn.lock b/yarn.lock index c4d07145e8..5e6dfdd242 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7782,22 +7782,11 @@ __metadata: "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" - "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" - "@elastic/elasticsearch": ^7.13.0 - "@elastic/elasticsearch-mock": ^1.0.0 - "@opensearch-project/opensearch": ^2.0.0 - "@short.io/opensearch-mock": ^0.3.1 - aws-os-connection: ^0.2.0 - aws-sdk: ^2.948.0 - elastic-builder: ^2.16.0 - lodash: ^4.17.21 msw: ^1.0.0 - uuid: ^8.3.2 - winston: ^3.2.1 languageName: unknown linkType: soft @@ -7832,26 +7821,12 @@ __metadata: "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/backend-test-utils": "workspace:^" - "@backstage/catalog-client": "workspace:^" - "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" - "@backstage/errors": "workspace:^" - "@backstage/integration": "workspace:^" - "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-explore-common": "workspace:^" - "@backstage/plugin-permission-common": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" - dockerode: ^3.3.1 - express: ^4.17.1 - express-promise-router: ^4.1.0 - fs-extra: 10.1.0 - knex: ^2.0.0 - lodash: ^4.17.21 - msw: ^1.0.0 node-fetch: ^2.6.7 - p-limit: ^3.1.0 winston: ^3.2.1 languageName: unknown linkType: soft @@ -7886,18 +7861,11 @@ __metadata: "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" - "@backstage/errors": "workspace:^" - "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/plugin-techdocs-node": "workspace:^" - dockerode: ^3.3.1 - express: ^4.17.1 - express-promise-router: ^4.1.0 - fs-extra: 10.1.0 - knex: ^2.0.0 lodash: ^4.17.21 msw: ^1.0.0 node-fetch: ^2.6.7 @@ -22582,9 +22550,7 @@ __metadata: version: 0.0.0-use.local resolution: "example-backend-next@workspace:packages/backend-next" dependencies: - "@backstage/backend-common": "workspace:^" "@backstage/backend-defaults": "workspace:^" - "@backstage/backend-plugin-api": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/plugin-app-backend": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" From fea29cf62ae0a12eef6d30c0d4d5fc29d7ea0875 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Thu, 9 Mar 2023 14:24:47 +0100 Subject: [PATCH 23/27] import from main entry point to avoid duplicate exports Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- .../search-backend-module-catalog/src/alpha.ts | 9 +-------- .../src/alpha.ts | 15 +-------------- .../search-backend-module-explore/src/alpha.ts | 9 +-------- .../search-backend-module-techdocs/src/alpha.ts | 4 +--- 4 files changed, 4 insertions(+), 33 deletions(-) diff --git a/plugins/search-backend-module-catalog/src/alpha.ts b/plugins/search-backend-module-catalog/src/alpha.ts index 89c7479035..c633dc71b2 100644 --- a/plugins/search-backend-module-catalog/src/alpha.ts +++ b/plugins/search-backend-module-catalog/src/alpha.ts @@ -27,16 +27,9 @@ import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; import { - CatalogCollatorEntityTransformer, DefaultCatalogCollatorFactory, DefaultCatalogCollatorFactoryOptions, -} from './collators'; - -export type { - CatalogCollatorEntityTransformer, - DefaultCatalogCollatorFactory, - DefaultCatalogCollatorFactoryOptions, -}; +} from '@backstage/plugin-search-backend-module-catalog'; /** * @alpha diff --git a/plugins/search-backend-module-elasticsearch/src/alpha.ts b/plugins/search-backend-module-elasticsearch/src/alpha.ts index d2ff1568f6..538124f004 100644 --- a/plugins/search-backend-module-elasticsearch/src/alpha.ts +++ b/plugins/search-backend-module-elasticsearch/src/alpha.ts @@ -21,14 +21,10 @@ import { loggerToWinstonLogger } from '@backstage/backend-common'; import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; import { - ElasticSearchHighlightConfig, - ElasticSearchQueryTranslatorOptions, - ElasticSearchConcreteQuery, ElasticSearchCustomIndexTemplate, - ElasticSearchCustomIndexTemplateBody, ElasticSearchQueryTranslator, ElasticSearchSearchEngine, -} from './engines'; +} from '@backstage/plugin-search-backend-module-elasticsearch'; /** * @alpha @@ -76,12 +72,3 @@ export const searchModuleElasticsearchEngine = createBackendModule( }, }), ); - -export type { - ElasticSearchCustomIndexTemplate, - ElasticSearchCustomIndexTemplateBody, - ElasticSearchQueryTranslator, - ElasticSearchQueryTranslatorOptions, - ElasticSearchHighlightConfig, - ElasticSearchConcreteQuery, -}; diff --git a/plugins/search-backend-module-explore/src/alpha.ts b/plugins/search-backend-module-explore/src/alpha.ts index 01303db631..f13c3c2406 100644 --- a/plugins/search-backend-module-explore/src/alpha.ts +++ b/plugins/search-backend-module-explore/src/alpha.ts @@ -28,16 +28,9 @@ import { loggerToWinstonLogger } from '@backstage/backend-common'; import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; import { - ToolDocument, ToolDocumentCollatorFactory, ToolDocumentCollatorFactoryOptions, -} from './collators'; - -export type { - ToolDocumentCollatorFactoryOptions, - ToolDocument, - ToolDocumentCollatorFactory, -}; +} from '@backstage/plugin-search-backend-module-explore'; /** * @alpha diff --git a/plugins/search-backend-module-techdocs/src/alpha.ts b/plugins/search-backend-module-techdocs/src/alpha.ts index c781ad40ca..1432cc88fb 100644 --- a/plugins/search-backend-module-techdocs/src/alpha.ts +++ b/plugins/search-backend-module-techdocs/src/alpha.ts @@ -30,9 +30,7 @@ import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-back import { DefaultTechDocsCollatorFactory, TechDocsCollatorFactoryOptions, -} from './collators'; - -export type { TechDocsCollatorFactoryOptions }; +} from '@backstage/plugin-search-backend-module-techdocs'; /** * @alpha From 05c132cbed911a125ceb73d176dd3d52f701f011 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Thu, 9 Mar 2023 14:27:46 +0100 Subject: [PATCH 24/27] rename type from IndexServiceBuildOptions to SearchIndexServiceStartOptions Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- plugins/search-backend-node/src/alpha.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/search-backend-node/src/alpha.ts b/plugins/search-backend-node/src/alpha.ts index 22c6508b15..1565b6002d 100644 --- a/plugins/search-backend-node/src/alpha.ts +++ b/plugins/search-backend-node/src/alpha.ts @@ -39,7 +39,7 @@ import { IndexBuilder } from './IndexBuilder'; * @alpha * Options for build method on {@link SearchIndexService}. */ -export type IndexServiceBuildOptions = { +export type SearchIndexServiceStartOptions = { searchEngine: SearchEngine; collators: RegisterCollatorParameters[]; decorators: RegisterDecoratorParameters[]; @@ -53,7 +53,7 @@ export interface SearchIndexService { /** * Starts indexing process */ - start(options: IndexServiceBuildOptions): Promise; + start(options: SearchIndexServiceStartOptions): Promise; /** * Returns an index types list. */ @@ -97,7 +97,7 @@ class DefaultSearchIndexService implements SearchIndexService { return new DefaultSearchIndexService(options); } - async start(options: IndexServiceBuildOptions): Promise { + async start(options: SearchIndexServiceStartOptions): Promise { this.indexBuilder = new IndexBuilder({ logger: this.logger, searchEngine: options.searchEngine, From fde0f1951ad749c0874c36db924d2172bcc26c09 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Thu, 9 Mar 2023 14:44:37 +0100 Subject: [PATCH 25/27] fix exports of deprecated collators/types Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- plugins/catalog-backend/src/index.ts | 29 ++++++++++++++++++-- plugins/explore-backend/src/index.ts | 28 +++++++++++++------ plugins/techdocs-backend/src/search/index.ts | 15 ++++++---- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index 594b96b347..0a209def82 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -37,7 +37,30 @@ import { * @deprecated * import from @backstage/search-backend-module-catalog instead */ -export type { - DefaultCatalogCollatorFactoryOptions, - CatalogCollatorEntityTransformer, +export const DefaultCatalogCollatorFactory = _DefaultCatalogCollatorFactory; + +/** + * @deprecated + * import from @backstage/search-backend-module-catalog instead + */ +export const defaultCatalogCollatorEntityTransformer = + _defaultCatalogCollatorEntityTransformer; + +import type { + DefaultCatalogCollatorFactoryOptions as _DefaultCatalogCollatorFactoryOptions, + CatalogCollatorEntityTransformer as _CatalogCollatorEntityTransformer, } from '@backstage/plugin-search-backend-module-catalog'; + +/** + * @deprecated + * import from @backstage/search-backend-module-catalog instead + */ +export type DefaultCatalogCollatorFactoryOptions = + _DefaultCatalogCollatorFactoryOptions; + +/** + * @deprecated + * import from @backstage/search-backend-module-catalog instead + */ +export type CatalogCollatorEntityTransformer = + _CatalogCollatorEntityTransformer; diff --git a/plugins/explore-backend/src/index.ts b/plugins/explore-backend/src/index.ts index 6c456bffc2..335825e592 100644 --- a/plugins/explore-backend/src/index.ts +++ b/plugins/explore-backend/src/index.ts @@ -28,17 +28,27 @@ export * from './tools'; */ export { exampleTools } from './example/exampleTools'; -/** - * @deprecated - * import from @backstage/plugin-search-backend-module-explore instead - */ -export { ToolDocumentCollatorFactory } from '@backstage/plugin-search-backend-module-explore'; +import { ToolDocumentCollatorFactory as _ToolDocumentCollatorFactory } from '@backstage/plugin-search-backend-module-explore'; +import type { + ToolDocument as _ToolDocument, + ToolDocumentCollatorFactoryOptions as _ToolDocumentCollatorFactoryOptions, +} from '@backstage/plugin-search-backend-module-explore'; /** * @deprecated * import from @backstage/plugin-search-backend-module-explore instead */ -export type { - ToolDocument, - ToolDocumentCollatorFactoryOptions, -} from '@backstage/plugin-search-backend-module-explore'; +export const ToolDocumentCollatorFactory = _ToolDocumentCollatorFactory; + +/** + * @deprecated + * import from @backstage/plugin-search-backend-module-explore instead + */ +export type ToolDocument = _ToolDocument; + +/** + * @deprecated + * import from @backstage/plugin-search-backend-module-explore instead + */ +export type ToolDocumentCollatorFactoryOptions = + _ToolDocumentCollatorFactoryOptions; diff --git a/plugins/techdocs-backend/src/search/index.ts b/plugins/techdocs-backend/src/search/index.ts index 0c403ef0d7..bf0d61687d 100644 --- a/plugins/techdocs-backend/src/search/index.ts +++ b/plugins/techdocs-backend/src/search/index.ts @@ -20,14 +20,17 @@ export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator'; -/** - * @deprecated - * import from @backstage/search-backend-module-techdocs instead - */ -export type { TechDocsCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-techdocs'; +import { DefaultTechDocsCollatorFactory as _DefaultTechDocsCollatorFactory } from '@backstage/plugin-search-backend-module-techdocs'; +import type { TechDocsCollatorFactoryOptions as _TechDocsCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-techdocs'; /** * @deprecated * import from @backstage/search-backend-module-techdocs instead */ -export { DefaultTechDocsCollatorFactory } from '@backstage/plugin-search-backend-module-techdocs'; +export type TechDocsCollatorFactoryOptions = _TechDocsCollatorFactoryOptions; + +/** + * @deprecated + * import from @backstage/search-backend-module-techdocs instead + */ +export const DefaultTechDocsCollatorFactory = _DefaultTechDocsCollatorFactory; From ab61195ea9790ad01e28733421c58d9fcfb704d4 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Thu, 9 Mar 2023 15:02:08 +0100 Subject: [PATCH 26/27] update api reports Signed-off-by: Emma Indal Co-authored-by: Camila Loiola --- plugins/catalog-backend/api-report.md | 31 +++++------- plugins/catalog-backend/src/index.ts | 16 +++---- plugins/explore-backend/api-report.md | 16 ++++--- plugins/explore-backend/src/index.ts | 12 ++--- .../alpha-api-report.md | 48 ++----------------- .../alpha-api-report.md | 43 +---------------- .../alpha-api-report.md | 38 ++------------- .../alpha-api-report.md | 22 ++------- .../search-backend-node/alpha-api-report.md | 16 +++---- plugins/techdocs-backend/api-report.md | 10 ++-- plugins/techdocs-backend/src/search/index.ts | 8 ++-- 11 files changed, 66 insertions(+), 194 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 3d6a33fbf9..d491f391d5 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -11,7 +11,7 @@ import { AnalyzeLocationGenerateEntity as AnalyzeLocationGenerateEntity_2 } from import { AnalyzeLocationRequest as AnalyzeLocationRequest_2 } from '@backstage/plugin-catalog-common'; import { AnalyzeLocationResponse as AnalyzeLocationResponse_2 } from '@backstage/plugin-catalog-common'; import { CatalogApi } from '@backstage/catalog-client'; -import { CatalogCollatorEntityTransformer } from '@backstage/plugin-search-backend-module-catalog'; +import type { CatalogCollatorEntityTransformer as CatalogCollatorEntityTransformer_2 } from '@backstage/plugin-search-backend-module-catalog'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; import { CatalogProcessor as CatalogProcessor_2 } from '@backstage/plugin-catalog-node'; import { CatalogProcessorCache as CatalogProcessorCache_2 } from '@backstage/plugin-catalog-node'; @@ -24,15 +24,9 @@ import { CatalogProcessorRefreshKeysResult as CatalogProcessorRefreshKeysResult_ import { CatalogProcessorRelationResult as CatalogProcessorRelationResult_2 } from '@backstage/plugin-catalog-node'; import { CatalogProcessorResult as CatalogProcessorResult_2 } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; -<<<<<<< HEAD +import { DefaultCatalogCollatorFactory as DefaultCatalogCollatorFactory_2 } from '@backstage/plugin-search-backend-module-catalog'; +import type { DefaultCatalogCollatorFactoryOptions as DefaultCatalogCollatorFactoryOptions_2 } from '@backstage/plugin-search-backend-module-catalog'; import { DeferredEntity as DeferredEntity_2 } from '@backstage/plugin-catalog-node'; -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; -======= -import { defaultCatalogCollatorEntityTransformer } from '@backstage/plugin-search-backend-module-catalog'; -import { DefaultCatalogCollatorFactory } from '@backstage/plugin-search-backend-module-catalog'; -import { DefaultCatalogCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-catalog'; -import { DeferredEntity } from '@backstage/plugin-catalog-node'; ->>>>>>> 56a45401d3d (feat(search): create search collator modules) import { Entity } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import { EntityProvider as EntityProvider_2 } from '@backstage/plugin-catalog-node'; @@ -52,11 +46,6 @@ import { PermissionRule } from '@backstage/plugin-permission-node'; import { PermissionRuleParams } from '@backstage/plugin-permission-common'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; -<<<<<<< HEAD -import { Readable } from 'stream'; -======= -import { processingResult } from '@backstage/plugin-catalog-node'; ->>>>>>> 56a45401d3d (feat(search): create search collator modules) import { Router } from 'express'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { TokenManager } from '@backstage/backend-common'; @@ -172,7 +161,9 @@ export class CatalogBuilder { useLegacySingleProcessorValidation(): this; } -export { CatalogCollatorEntityTransformer }; +// @public @deprecated (undocumented) +export type CatalogCollatorEntityTransformer = + CatalogCollatorEntityTransformer_2; // @public (undocumented) export type CatalogEnvironment = { @@ -295,11 +286,15 @@ export class DefaultCatalogCollator { readonly visibilityPermission: Permission; } -export { defaultCatalogCollatorEntityTransformer }; +// @public @deprecated (undocumented) +export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer_2; -export { DefaultCatalogCollatorFactory }; +// @public @deprecated (undocumented) +export const DefaultCatalogCollatorFactory: typeof DefaultCatalogCollatorFactory_2; -export { DefaultCatalogCollatorFactoryOptions }; +// @public @deprecated (undocumented) +export type DefaultCatalogCollatorFactoryOptions = + DefaultCatalogCollatorFactoryOptions_2; // @public @deprecated (undocumented) export type DeferredEntity = DeferredEntity_2; diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index 0a209def82..eca155ce46 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -34,14 +34,14 @@ import { } from '@backstage/plugin-search-backend-module-catalog'; /** - * @deprecated - * import from @backstage/search-backend-module-catalog instead + * @public + * @deprecated import from `@backstage/search-backend-module-catalog` instead */ export const DefaultCatalogCollatorFactory = _DefaultCatalogCollatorFactory; /** - * @deprecated - * import from @backstage/search-backend-module-catalog instead + * @public + * @deprecated import from `@backstage/search-backend-module-catalog` instead */ export const defaultCatalogCollatorEntityTransformer = _defaultCatalogCollatorEntityTransformer; @@ -52,15 +52,15 @@ import type { } from '@backstage/plugin-search-backend-module-catalog'; /** - * @deprecated - * import from @backstage/search-backend-module-catalog instead + * @public + * @deprecated import from `@backstage/search-backend-module-catalog` instead */ export type DefaultCatalogCollatorFactoryOptions = _DefaultCatalogCollatorFactoryOptions; /** - * @deprecated - * import from @backstage/search-backend-module-catalog instead + * @public + * @deprecated import from `@backstage/search-backend-module-catalog` instead */ export type CatalogCollatorEntityTransformer = _CatalogCollatorEntityTransformer; diff --git a/plugins/explore-backend/api-report.md b/plugins/explore-backend/api-report.md index 4bcdc07079..e3933e350b 100644 --- a/plugins/explore-backend/api-report.md +++ b/plugins/explore-backend/api-report.md @@ -8,9 +8,9 @@ import express from 'express'; import { GetExploreToolsRequest } from '@backstage/plugin-explore-common'; import { GetExploreToolsResponse } from '@backstage/plugin-explore-common'; import { Logger } from 'winston'; -import { ToolDocument } from '@backstage/plugin-search-backend-module-explore'; -import { ToolDocumentCollatorFactory } from '@backstage/plugin-search-backend-module-explore'; -import { ToolDocumentCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-explore'; +import type { ToolDocument as ToolDocument_2 } from '@backstage/plugin-search-backend-module-explore'; +import { ToolDocumentCollatorFactory as ToolDocumentCollatorFactory_2 } from '@backstage/plugin-search-backend-module-explore'; +import type { ToolDocumentCollatorFactoryOptions as ToolDocumentCollatorFactoryOptions_2 } from '@backstage/plugin-search-backend-module-explore'; // @public (undocumented) export function createRouter(options: RouterOptions): Promise; @@ -36,9 +36,13 @@ export class StaticExploreToolProvider implements ExploreToolProvider { getTools(request: GetExploreToolsRequest): Promise; } -export { ToolDocument }; +// @public @deprecated (undocumented) +export type ToolDocument = ToolDocument_2; -export { ToolDocumentCollatorFactory }; +// @public @deprecated (undocumented) +export const ToolDocumentCollatorFactory: typeof ToolDocumentCollatorFactory_2; -export { ToolDocumentCollatorFactoryOptions }; +// @public @deprecated (undocumented) +export type ToolDocumentCollatorFactoryOptions = + ToolDocumentCollatorFactoryOptions_2; ``` diff --git a/plugins/explore-backend/src/index.ts b/plugins/explore-backend/src/index.ts index 335825e592..ca501db96b 100644 --- a/plugins/explore-backend/src/index.ts +++ b/plugins/explore-backend/src/index.ts @@ -35,20 +35,20 @@ import type { } from '@backstage/plugin-search-backend-module-explore'; /** - * @deprecated - * import from @backstage/plugin-search-backend-module-explore instead + * @public + * @deprecated import from `@backstage/search-backend-module-explore` instead */ export const ToolDocumentCollatorFactory = _ToolDocumentCollatorFactory; /** - * @deprecated - * import from @backstage/plugin-search-backend-module-explore instead + * @public + * @deprecated import from `@backstage/search-backend-module-explore` instead */ export type ToolDocument = _ToolDocument; /** - * @deprecated - * import from @backstage/plugin-search-backend-module-explore instead + * @public + * @deprecated import from `@backstage/search-backend-module-explore` instead */ export type ToolDocumentCollatorFactoryOptions = _ToolDocumentCollatorFactoryOptions; diff --git a/plugins/search-backend-module-catalog/alpha-api-report.md b/plugins/search-backend-module-catalog/alpha-api-report.md index e0173fce74..70fae9ef93 100644 --- a/plugins/search-backend-module-catalog/alpha-api-report.md +++ b/plugins/search-backend-module-catalog/alpha-api-report.md @@ -3,55 +3,13 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -/// - import { BackendFeature } from '@backstage/backend-plugin-api'; -import { CatalogApi } from '@backstage/catalog-client'; -import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; -import { Config } from '@backstage/config'; -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; -import { Entity } from '@backstage/catalog-model'; -import { GetEntitiesRequest } from '@backstage/catalog-client'; -import { Permission } from '@backstage/plugin-permission-common'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; -import { Readable } from 'stream'; +import { DefaultCatalogCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-catalog'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; -import { TokenManager } from '@backstage/backend-common'; - -// @public (undocumented) -export type CatalogCollatorEntityTransformer = ( - entity: Entity, -) => Omit; - -// @public (undocumented) -export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { - // (undocumented) - static fromConfig( - _config: Config, - options: DefaultCatalogCollatorFactoryOptions, - ): DefaultCatalogCollatorFactory; - // (undocumented) - getCollator(): Promise; - // (undocumented) - readonly type = 'software-catalog'; - // (undocumented) - readonly visibilityPermission: Permission; -} - -// @public (undocumented) -export type DefaultCatalogCollatorFactoryOptions = { - discovery: PluginEndpointDiscovery; - tokenManager: TokenManager; - locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; - batchSize?: number; - catalogClient?: CatalogApi; - entityTransformer?: CatalogCollatorEntityTransformer; -}; // @alpha export const searchModuleCatalogCollator: ( - options: SearchModuleCatalogCollatorOptions, + options?: SearchModuleCatalogCollatorOptions | undefined, ) => BackendFeature; // @alpha @@ -59,7 +17,7 @@ export type SearchModuleCatalogCollatorOptions = Omit< DefaultCatalogCollatorFactoryOptions, 'discovery' | 'tokenManager' > & { - schedule: TaskScheduleDefinition; + schedule?: TaskScheduleDefinition; }; // (No @packageDocumentation comment for this package) diff --git a/plugins/search-backend-module-elasticsearch/alpha-api-report.md b/plugins/search-backend-module-elasticsearch/alpha-api-report.md index fca920f3d1..91007c20de 100644 --- a/plugins/search-backend-module-elasticsearch/alpha-api-report.md +++ b/plugins/search-backend-module-elasticsearch/alpha-api-report.md @@ -4,47 +4,8 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; -import { SearchQuery } from '@backstage/plugin-search-common'; - -// @public -export type ElasticSearchConcreteQuery = { - documentTypes?: string[]; - elasticSearchQuery: Object; - pageSize: number; -}; - -// @public -export type ElasticSearchCustomIndexTemplate = { - name: string; - body: ElasticSearchCustomIndexTemplateBody; -}; - -// @public -export type ElasticSearchCustomIndexTemplateBody = { - index_patterns: string[]; - composed_of?: string[]; - template?: Record; -}; - -// @public (undocumented) -export type ElasticSearchHighlightConfig = { - fragmentDelimiter: string; - fragmentSize: number; - numFragments: number; - preTag: string; - postTag: string; -}; - -// @public -export type ElasticSearchQueryTranslator = ( - query: SearchQuery, - options?: ElasticSearchQueryTranslatorOptions, -) => ElasticSearchConcreteQuery; - -// @public -export type ElasticSearchQueryTranslatorOptions = { - highlightOptions?: ElasticSearchHighlightConfig; -}; +import { ElasticSearchCustomIndexTemplate } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { ElasticSearchQueryTranslator } from '@backstage/plugin-search-backend-module-elasticsearch'; // @alpha export const searchModuleElasticsearchEngine: ( diff --git a/plugins/search-backend-module-explore/alpha-api-report.md b/plugins/search-backend-module-explore/alpha-api-report.md index 5c5189c8bd..cf32b59dd3 100644 --- a/plugins/search-backend-module-explore/alpha-api-report.md +++ b/plugins/search-backend-module-explore/alpha-api-report.md @@ -3,21 +3,13 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -/// - import { BackendFeature } from '@backstage/backend-plugin-api'; -import { Config } from '@backstage/config'; -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; -import { ExploreTool } from '@backstage/plugin-explore-common'; -import { IndexableDocument } from '@backstage/plugin-search-common'; -import { Logger } from 'winston'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; -import { Readable } from 'stream'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { ToolDocumentCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-explore'; // @alpha export const searchModuleExploreCollator: ( - options: SearchModuleExploreCollatorOptions, + options?: SearchModuleExploreCollatorOptions | undefined, ) => BackendFeature; // @alpha @@ -25,31 +17,7 @@ export type SearchModuleExploreCollatorOptions = Omit< ToolDocumentCollatorFactoryOptions, 'logger' | 'discovery' > & { - schedule: TaskScheduleDefinition; -}; - -// @public -export interface ToolDocument extends IndexableDocument, ExploreTool {} - -// @public -export class ToolDocumentCollatorFactory implements DocumentCollatorFactory { - // (undocumented) - execute(): AsyncGenerator; - // (undocumented) - static fromConfig( - _config: Config, - options: ToolDocumentCollatorFactoryOptions, - ): ToolDocumentCollatorFactory; - // (undocumented) - getCollator(): Promise; - // (undocumented) - readonly type: string; -} - -// @public -export type ToolDocumentCollatorFactoryOptions = { - discovery: PluginEndpointDiscovery; - logger: Logger; + schedule?: TaskScheduleDefinition; }; // (No @packageDocumentation comment for this package) diff --git a/plugins/search-backend-module-techdocs/alpha-api-report.md b/plugins/search-backend-module-techdocs/alpha-api-report.md index 3360055c62..71ce09b744 100644 --- a/plugins/search-backend-module-techdocs/alpha-api-report.md +++ b/plugins/search-backend-module-techdocs/alpha-api-report.md @@ -3,18 +3,13 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -/// - import { BackendFeature } from '@backstage/backend-plugin-api'; -import { CatalogApi } from '@backstage/catalog-client'; -import { Logger } from 'winston'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; -import { TokenManager } from '@backstage/backend-common'; +import { TechDocsCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-techdocs'; // @alpha export const searchModuleTechDocsCollator: ( - options: SearchModuleTechDocsCollatorOptions, + options?: SearchModuleTechDocsCollatorOptions | undefined, ) => BackendFeature; // @alpha @@ -22,18 +17,7 @@ export type SearchModuleTechDocsCollatorOptions = Omit< TechDocsCollatorFactoryOptions, 'logger' | 'discovery' | 'tokenManager' > & { - schedule: TaskScheduleDefinition; -}; - -// @public -export type TechDocsCollatorFactoryOptions = { - discovery: PluginEndpointDiscovery; - logger: Logger; - tokenManager: TokenManager; - locationTemplate?: string; - catalogClient?: CatalogApi; - parallelismLimit?: number; - legacyPathCasing?: boolean; + schedule?: TaskScheduleDefinition; }; // (No @packageDocumentation comment for this package) diff --git a/plugins/search-backend-node/alpha-api-report.md b/plugins/search-backend-node/alpha-api-report.md index 27d4875bd6..9dd754267d 100644 --- a/plugins/search-backend-node/alpha-api-report.md +++ b/plugins/search-backend-node/alpha-api-report.md @@ -11,13 +11,6 @@ import { SearchEngine } from '@backstage/plugin-search-common'; import { ServiceRef } from '@backstage/backend-plugin-api'; import { TaskRunner } from '@backstage/backend-tasks'; -// @alpha -export type IndexServiceBuildOptions = { - searchEngine: SearchEngine; - collators: RegisterCollatorParameters[]; - decorators: RegisterDecoratorParameters[]; -}; - // @public export interface RegisterCollatorParameters { factory: DocumentCollatorFactory; @@ -52,11 +45,18 @@ export const searchIndexRegistryExtensionPoint: ExtensionPoint; - start(options: IndexServiceBuildOptions): Promise; + start(options: SearchIndexServiceStartOptions): Promise; } // @alpha export const searchIndexServiceRef: ServiceRef; +// @alpha +export type SearchIndexServiceStartOptions = { + searchEngine: SearchEngine; + collators: RegisterCollatorParameters[]; + decorators: RegisterDecoratorParameters[]; +}; + // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 2abd9783f7..b8974e6d4f 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -6,7 +6,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { CatalogClient } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; -import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-search-backend-module-techdocs'; +import { DefaultTechDocsCollatorFactory as DefaultTechDocsCollatorFactory_2 } from '@backstage/plugin-search-backend-module-techdocs'; import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { GeneratorBuilder } from '@backstage/plugin-techdocs-node'; @@ -17,7 +17,7 @@ import { PluginCacheManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PreparerBuilder } from '@backstage/plugin-techdocs-node'; import { PublisherBase } from '@backstage/plugin-techdocs-node'; -import { TechDocsCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-techdocs'; +import type { TechDocsCollatorFactoryOptions as TechDocsCollatorFactoryOptions_2 } from '@backstage/plugin-search-backend-module-techdocs'; import { TechDocsDocument } from '@backstage/plugin-techdocs-node'; import { TokenManager } from '@backstage/backend-common'; import * as winston from 'winston'; @@ -45,7 +45,8 @@ export class DefaultTechDocsCollator { readonly visibilityPermission: Permission; } -export { DefaultTechDocsCollatorFactory }; +// @public @deprecated (undocumented) +export const DefaultTechDocsCollatorFactory: typeof DefaultTechDocsCollatorFactory_2; // @public export interface DocsBuildStrategy { @@ -90,7 +91,8 @@ export type ShouldBuildParameters = { entity: Entity; }; -export { TechDocsCollatorFactoryOptions }; +// @public @deprecated (undocumented) +export type TechDocsCollatorFactoryOptions = TechDocsCollatorFactoryOptions_2; // @public export type TechDocsCollatorOptions = { diff --git a/plugins/techdocs-backend/src/search/index.ts b/plugins/techdocs-backend/src/search/index.ts index bf0d61687d..fc2b48375e 100644 --- a/plugins/techdocs-backend/src/search/index.ts +++ b/plugins/techdocs-backend/src/search/index.ts @@ -24,13 +24,13 @@ import { DefaultTechDocsCollatorFactory as _DefaultTechDocsCollatorFactory } fro import type { TechDocsCollatorFactoryOptions as _TechDocsCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-techdocs'; /** - * @deprecated - * import from @backstage/search-backend-module-techdocs instead + * @public + * @deprecated import from `@backstage/search-backend-module-techdocs` instead */ export type TechDocsCollatorFactoryOptions = _TechDocsCollatorFactoryOptions; /** - * @deprecated - * import from @backstage/search-backend-module-techdocs instead + * @public + * @deprecated import from `@backstage/search-backend-module-techdocs` instead */ export const DefaultTechDocsCollatorFactory = _DefaultTechDocsCollatorFactory; From 9d667467374a808c289b72ff082ed73d973b696a Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Mon, 20 Mar 2023 13:16:34 +0100 Subject: [PATCH 27/27] Update readme files and api reports Co-authored-by: Emma Indal Signed-off-by: Camila Belo --- .changeset/tall-chairs-explain.md | 2 +- .changeset/wet-lamps-happen.md | 2 +- docs/features/search/how-to-guides.md | 2 +- .../search-backend-module-catalog/README.md | 18 ++++++++++++++++-- .../search-backend-module-explore/README.md | 18 ++++++++++++++++-- .../search-backend-module-techdocs/README.md | 18 ++++++++++++++++-- .../search-backend-node/alpha-api-report.md | 16 ++-------------- plugins/search-backend-node/src/alpha.ts | 10 +--------- 8 files changed, 54 insertions(+), 32 deletions(-) diff --git a/.changeset/tall-chairs-explain.md b/.changeset/tall-chairs-explain.md index e7367d2d4a..9709f225b4 100644 --- a/.changeset/tall-chairs-explain.md +++ b/.changeset/tall-chairs-explain.md @@ -4,4 +4,4 @@ '@backstage/plugin-explore-backend': patch --- -Deprecate the collator files in the backend package by extracting them into a separate module. +Collator factories instantiated in new backend system modules and now marked as deprecated. Will be continued to be exported publicly until the new backend system is fully rolled out. diff --git a/.changeset/wet-lamps-happen.md b/.changeset/wet-lamps-happen.md index 76e36d219d..066b34b23c 100644 --- a/.changeset/wet-lamps-happen.md +++ b/.changeset/wet-lamps-happen.md @@ -4,4 +4,4 @@ '@backstage/plugin-search-backend-module-explore': minor --- -Extend the index with custom collators by creating search plugin modules. +Package introduced to export search backend modules that can be used with the new backend system to extend search with plugin specific functionality, such as collators. For documentation on how to migrate, check out the [how to migrate to the new backend system guide](../docs/features/search/how-to-guides.md#how-to-migrate-to-use-search-together-with-the-new-backend-system). diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index 161ff6844c..abb29ab4eb 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -370,7 +370,7 @@ export const SearchModal = ({ toggleModal }: { toggleModal: () => void }) => ( There are other more specific search results layout components that also accept result item extensions, check their documentation: [SearchResultList](https://backstage.io/storybook/?path=/story/plugins-search-searchresultlist--with-result-item-extensions) and [SearchResultGroup](https://backstage.io/storybook/?path=/story/plugins-search-searchresultgroup--with-result-item-extensions). -## How to migrate to use Search together with the new backend system +## How to migrate your backend installation to use Search together with the new backend system > DISCLAIMER: The new backend system is in alpha, and so are the search backend support for the new backend system. We don't recommend you to migrate your backend installations to the new system yet. But if you want to experiment, this is the guide for you! diff --git a/plugins/search-backend-module-catalog/README.md b/plugins/search-backend-module-catalog/README.md index b6c734d893..b5960e2789 100644 --- a/plugins/search-backend-module-catalog/README.md +++ b/plugins/search-backend-module-catalog/README.md @@ -6,6 +6,22 @@ This package exports catalog backend modules responsible for extending search. ## Example +### Use default schedule + +```tsx +// packages/backend-next/src/index.ts +import { createBackend } from '@backstage/backend-defaults'; +import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; +import { searchModuleCatalogCollator } from '@backstage/plugin-search-backend-module-catalog/alpha'; + +const backend = createBackend(); +backend.add(searchPlugin()); +backend.add(searchModuleCatalogCollator()); +backend.start(); +``` + +### Use custom schedule + ```tsx // packages/backend-next/src/index.ts import { createBackend } from '@backstage/backend-defaults'; @@ -23,5 +39,3 @@ backend.add(searchPlugin()); backend.add(searchModuleCatalogCollator({ schedule })); backend.start(); ``` - -... diff --git a/plugins/search-backend-module-explore/README.md b/plugins/search-backend-module-explore/README.md index d2b00767e2..a87f2d78f9 100644 --- a/plugins/search-backend-module-explore/README.md +++ b/plugins/search-backend-module-explore/README.md @@ -6,6 +6,22 @@ This package exports explore backend modules responsible for extending search. ## Example +### Use default schedule + +```tsx +// packages/backend-next/src/index.ts +import { createBackend } from '@backstage/backend-defaults'; +import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; +import { searchModuleExploreCollator } from '@backstage/plugin-search-backend-module-explore/alpha'; + +const backend = createBackend(); +backend.add(searchPlugin()); +backend.add(searchModuleExploreCollator()); +backend.start(); +``` + +### Use custom schedule + ```tsx // packages/backend-next/src/index.ts import { createBackend } from '@backstage/backend-defaults'; @@ -23,5 +39,3 @@ backend.add(searchPlugin()); backend.add(searchModuleExploreCollator({ schedule })); backend.start(); ``` - -... diff --git a/plugins/search-backend-module-techdocs/README.md b/plugins/search-backend-module-techdocs/README.md index 817bb40998..5de310466b 100644 --- a/plugins/search-backend-module-techdocs/README.md +++ b/plugins/search-backend-module-techdocs/README.md @@ -6,6 +6,22 @@ This package exports techdocs backend modules responsible for extending search. ## Example +### Use default schedule + +```tsx +// packages/backend-next/src/index.ts +import { createBackend } from '@backstage/backend-defaults'; +import { searchPlugin } from '@backstage/plugin-search-backend/alpha'; +import { searchModuleTechDocsCollator } from '@backstage/plugin-search-backend-module-techdocs/alpha'; + +const backend = createBackend(); +backend.add(searchPlugin()); +backend.add(searchModuleTechDocsCollator()); +backend.start(); +``` + +### Use custom schedule + ```tsx // packages/backend-next/src/index.ts import { createBackend } from '@backstage/backend-defaults'; @@ -23,5 +39,3 @@ backend.add(searchPlugin()); backend.add(searchModuleTechDocsCollator({ schedule })); backend.start(); ``` - -... diff --git a/plugins/search-backend-node/alpha-api-report.md b/plugins/search-backend-node/alpha-api-report.md index 9dd754267d..74b7a4b4bc 100644 --- a/plugins/search-backend-node/alpha-api-report.md +++ b/plugins/search-backend-node/alpha-api-report.md @@ -3,24 +3,12 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; -import { DocumentDecoratorFactory } from '@backstage/plugin-search-common'; import { DocumentTypeInfo } from '@backstage/plugin-search-common'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; +import { RegisterCollatorParameters } from '@backstage/plugin-search-backend-node'; +import { RegisterDecoratorParameters } from '@backstage/plugin-search-backend-node'; import { SearchEngine } from '@backstage/plugin-search-common'; import { ServiceRef } from '@backstage/backend-plugin-api'; -import { TaskRunner } from '@backstage/backend-tasks'; - -// @public -export interface RegisterCollatorParameters { - factory: DocumentCollatorFactory; - schedule: TaskRunner; -} - -// @public -export interface RegisterDecoratorParameters { - factory: DocumentDecoratorFactory; -} // @alpha export interface SearchEngineRegistryExtensionPoint { diff --git a/plugins/search-backend-node/src/alpha.ts b/plugins/search-backend-node/src/alpha.ts index 1565b6002d..943e11b580 100644 --- a/plugins/search-backend-node/src/alpha.ts +++ b/plugins/search-backend-node/src/alpha.ts @@ -31,7 +31,7 @@ import { createExtensionPoint } from '@backstage/backend-plugin-api'; import { RegisterCollatorParameters, RegisterDecoratorParameters, -} from './types'; +} from '@backstage/plugin-search-backend-node'; import { IndexBuilder } from './IndexBuilder'; @@ -157,11 +157,3 @@ export const searchIndexRegistryExtensionPoint = createExtensionPoint({ id: 'search.index.registry', }); - -/** - * @alpha - */ -export type { - RegisterCollatorParameters, - RegisterDecoratorParameters, -} from './types';