diff --git a/.changeset/metal-kings-argue.md b/.changeset/metal-kings-argue.md new file mode 100644 index 0000000000..0019139362 --- /dev/null +++ b/.changeset/metal-kings-argue.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-catalog': patch +--- + +Breaking change in the alpha export `searchModuleCatalogCollator`: Moved collator settings from module options into app-config. You are now expected to set up the catalog collator under the `search.collators.catalog` configuration key. There is also a new `catalogCollatorExtensionPoint` extension point for the module, wherein you can set custom transformers. diff --git a/plugins/search-backend-module-catalog/alpha-api-report.md b/plugins/search-backend-module-catalog/alpha-api-report.md index 7c6e91b6a1..658c1aec15 100644 --- a/plugins/search-backend-module-catalog/alpha-api-report.md +++ b/plugins/search-backend-module-catalog/alpha-api-report.md @@ -4,21 +4,19 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; -import { DefaultCatalogCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-catalog'; -import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { CatalogCollatorEntityTransformer } from '@backstage/plugin-search-backend-module-catalog'; +import { ExtensionPoint } from '@backstage/backend-plugin-api'; // @alpha -export const searchModuleCatalogCollator: ( - options?: SearchModuleCatalogCollatorOptions | undefined, -) => BackendFeature; - -// @alpha -export type SearchModuleCatalogCollatorOptions = Omit< - DefaultCatalogCollatorFactoryOptions, - 'discovery' | 'tokenManager' | 'catalogClient' -> & { - schedule?: TaskScheduleDefinition; +export type CatalogCollatorExtensionPoint = { + setEntityTransformer(transformer: CatalogCollatorEntityTransformer): void; }; +// @alpha +export const catalogCollatorExtensionPoint: ExtensionPoint; + +// @alpha +export const searchModuleCatalogCollator: () => BackendFeature; + // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/search-backend-module-catalog/api-report.md b/plugins/search-backend-module-catalog/api-report.md index b773860fe8..6f5813ca9e 100644 --- a/plugins/search-backend-module-catalog/api-report.md +++ b/plugins/search-backend-module-catalog/api-report.md @@ -24,11 +24,11 @@ export type CatalogCollatorEntityTransformer = ( // @public (undocumented) export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer; -// @public (undocumented) +// @public export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { // (undocumented) static fromConfig( - _config: Config, + configRoot: Config, options: DefaultCatalogCollatorFactoryOptions, ): DefaultCatalogCollatorFactory; // (undocumented) diff --git a/plugins/search-backend-module-catalog/config.d.ts b/plugins/search-backend-module-catalog/config.d.ts new file mode 100644 index 0000000000..0105764afb --- /dev/null +++ b/plugins/search-backend-module-catalog/config.d.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + +export interface Config { + search?: { + collators?: { + /** + * Configuration options for `@backstage/plugin-search-backend-module-catalog` + */ + catalog?: { + /** + * A templating string with placeholders, to form the final location of + * the entity. + * + * Defaults to '/catalog/:namespace/:kind/:name' + */ + locationTemplate?: string; + /** + * A filter expression passed to the catalog client, to select what + * entities to collate. + * + * Defaults to no filter, ie indexing all entities. + */ + filter?: object; + /** + * The number of entities to process at a time. Keep this at a + * reasonable number to avoid overloading either the catalog or the + * search backend. + * + * Defaults to 500 + */ + batchSize?: number; + /** + * The schedule for how often to run the collation job. + */ + schedule?: TaskScheduleDefinitionConfig; + }; + }; + }; +} diff --git a/plugins/search-backend-module-catalog/package.json b/plugins/search-backend-module-catalog/package.json index f279ec399d..1d9ee95bae 100644 --- a/plugins/search-backend-module-catalog/package.json +++ b/plugins/search-backend-module-catalog/package.json @@ -48,6 +48,7 @@ "@backstage/catalog-client": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", @@ -61,6 +62,8 @@ "msw": "^1.0.0" }, "files": [ - "dist" - ] + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" } diff --git a/plugins/search-backend-module-catalog/src/alpha.test.ts b/plugins/search-backend-module-catalog/src/alpha.test.ts index 8a3082c6aa..edd922b7fb 100644 --- a/plugins/search-backend-module-catalog/src/alpha.test.ts +++ b/plugins/search-backend-module-catalog/src/alpha.test.ts @@ -14,17 +14,11 @@ * limitations under the License. */ -import { startTestBackend } from '@backstage/backend-test-utils'; +import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; import { searchModuleCatalogCollator } from './alpha'; describe('searchModuleCatalogCollator', () => { - const schedule = { - frequency: { minutes: 10 }, - timeout: { minutes: 15 }, - initialDelay: { seconds: 3 }, - }; - it('should register the catalog collator to the search index registry extension point with factory and schedule', async () => { const extensionPointMock = { addCollator: jest.fn(), @@ -34,7 +28,7 @@ describe('searchModuleCatalogCollator', () => { extensionPoints: [ [searchIndexRegistryExtensionPoint, extensionPointMock], ], - features: [searchModuleCatalogCollator({ schedule })], + features: [searchModuleCatalogCollator()], }); expect(extensionPointMock.addCollator).toHaveBeenCalledTimes(1); @@ -43,4 +37,37 @@ describe('searchModuleCatalogCollator', () => { schedule: expect.objectContaining({ run: expect.any(Function) }), }); }); + + it('refuses to start up with a broken schedule', async () => { + await expect( + startTestBackend({ + extensionPoints: [ + [ + searchIndexRegistryExtensionPoint, + { + addCollator: jest.fn(), + }, + ], + ], + features: [searchModuleCatalogCollator()], + services: [ + mockServices.rootConfig.factory({ + data: { + search: { + collators: { + catalog: { + schedule: { + frequency: { minutes: 10 }, + timeout: { minutes: 'boo' }, + initialDelay: { seconds: 3 }, + }, + }, + }, + }, + }, + }), + ], + }), + ).rejects.toThrow(/Invalid schedule/); + }); }); diff --git a/plugins/search-backend-module-catalog/src/alpha.ts b/plugins/search-backend-module-catalog/src/alpha.ts index cfd8c13539..0edf90fc7f 100644 --- a/plugins/search-backend-module-catalog/src/alpha.ts +++ b/plugins/search-backend-module-catalog/src/alpha.ts @@ -22,73 +22,89 @@ import { coreServices, createBackendModule, + createExtensionPoint, } from '@backstage/backend-plugin-api'; -import { TaskScheduleDefinition } from '@backstage/backend-tasks'; -import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; - -import { - DefaultCatalogCollatorFactory, - DefaultCatalogCollatorFactoryOptions, -} from '@backstage/plugin-search-backend-module-catalog'; import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha'; +import { + CatalogCollatorEntityTransformer, + DefaultCatalogCollatorFactory, +} from '@backstage/plugin-search-backend-module-catalog'; +import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; +import { readScheduleConfigOptions } from './collators/config'; /** + * Options for {@link catalogCollatorExtensionPoint}. + * * @alpha - * Options for {@link searchModuleCatalogCollator}. */ -export type SearchModuleCatalogCollatorOptions = Omit< - DefaultCatalogCollatorFactoryOptions, - 'discovery' | 'tokenManager' | 'catalogClient' -> & { - schedule?: TaskScheduleDefinition; +export type CatalogCollatorExtensionPoint = { + /** + * Allows you to customize how entities are shaped into documents. + */ + setEntityTransformer(transformer: CatalogCollatorEntityTransformer): void; }; +/** + * Extension point for customizing how catalog entities are shaped into + * documents for the search backend, when using + * {@link searchModuleCatalogCollator}. + * + * @alpha + */ +export const catalogCollatorExtensionPoint = + createExtensionPoint({ + id: 'search.catalogCollator.extension', + }); + /** * Search backend module for the Catalog index. * * @alpha */ -export const searchModuleCatalogCollator = createBackendModule( - (options?: SearchModuleCatalogCollatorOptions) => ({ - moduleId: 'catalogCollator', - pluginId: 'search', - register(env) { - env.registerInit({ - deps: { - config: coreServices.rootConfig, - discovery: coreServices.discovery, - tokenManager: coreServices.tokenManager, - scheduler: coreServices.scheduler, - indexRegistry: searchIndexRegistryExtensionPoint, - catalog: catalogServiceRef, - }, - async init({ - config, - discovery, - tokenManager, - scheduler, - indexRegistry, - catalog, - }) { - const defaultSchedule = { - frequency: { minutes: 10 }, - timeout: { minutes: 15 }, - initialDelay: { seconds: 3 }, - }; +export const searchModuleCatalogCollator = createBackendModule({ + moduleId: 'catalogCollator', + pluginId: 'search', + register(env) { + let entityTransformer: CatalogCollatorEntityTransformer | undefined; - indexRegistry.addCollator({ - schedule: scheduler.createScheduledTaskRunner( - options?.schedule ?? defaultSchedule, - ), - factory: DefaultCatalogCollatorFactory.fromConfig(config, { - ...options, - discovery, - tokenManager, - catalogClient: catalog, - }), - }); - }, - }); - }, - }), -); + env.registerExtensionPoint(catalogCollatorExtensionPoint, { + setEntityTransformer(transformer) { + if (entityTransformer) { + throw new Error('setEntityTransformer can only be called once'); + } + entityTransformer = transformer; + }, + }); + + env.registerInit({ + deps: { + config: coreServices.rootConfig, + discovery: coreServices.discovery, + tokenManager: coreServices.tokenManager, + scheduler: coreServices.scheduler, + indexRegistry: searchIndexRegistryExtensionPoint, + catalog: catalogServiceRef, + }, + async init({ + config, + discovery, + tokenManager, + scheduler, + indexRegistry, + catalog, + }) { + indexRegistry.addCollator({ + schedule: scheduler.createScheduledTaskRunner( + readScheduleConfigOptions(config), + ), + factory: DefaultCatalogCollatorFactory.fromConfig(config, { + entityTransformer, + discovery, + tokenManager, + catalogClient: catalog, + }), + }); + }, + }); + }, +}); diff --git a/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.test.ts b/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.test.ts index 462e777dff..7af3a8e9a2 100644 --- a/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.test.ts +++ b/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { PluginEndpointDiscovery, TokenManager, diff --git a/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.ts b/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.ts index ed81d39bbb..586346c385 100644 --- a/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.ts +++ b/plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; import { PluginEndpointDiscovery, TokenManager, @@ -24,27 +23,45 @@ import { CatalogClient, GetEntitiesRequest, } from '@backstage/catalog-client'; -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; -import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common/alpha'; +import { stringifyEntityRef } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; +import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common/alpha'; import { Permission } from '@backstage/plugin-permission-common'; +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; import { Readable } from 'stream'; import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; +import { readCollatorConfigOptions } from './config'; import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; -import { stringifyEntityRef } from '@backstage/catalog-model'; /** @public */ export type DefaultCatalogCollatorFactoryOptions = { discovery: PluginEndpointDiscovery; tokenManager: TokenManager; + /** + * @deprecated Use the config key `search.collators.catalog.locationTemplate` instead. + */ locationTemplate?: string; + /** + * @deprecated Use the config key `search.collators.catalog.filter` instead. + */ filter?: GetEntitiesRequest['filter']; + /** + * @deprecated Use the config key `search.collators.catalog.batchSize` instead. + */ batchSize?: number; catalogClient?: CatalogApi; + /** + * Allows you to customize how entities are shaped into documents. + */ entityTransformer?: CatalogCollatorEntityTransformer; }; -/** @public */ +/** + * Collates entities from the Catalog into documents for the search backend. + * + * @public + */ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { public readonly type = 'software-catalog'; public readonly visibilityPermission: Permission = @@ -58,13 +75,31 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { private entityTransformer: CatalogCollatorEntityTransformer; static fromConfig( - _config: Config, + configRoot: Config, options: DefaultCatalogCollatorFactoryOptions, ) { - return new DefaultCatalogCollatorFactory(options); + const configOptions = readCollatorConfigOptions(configRoot); + return new DefaultCatalogCollatorFactory({ + locationTemplate: + options.locationTemplate ?? configOptions.locationTemplate, + filter: options.filter ?? configOptions.filter, + batchSize: options.batchSize ?? configOptions.batchSize, + entityTransformer: options.entityTransformer, + discovery: options.discovery, + tokenManager: options.tokenManager, + catalogClient: options.catalogClient, + }); } - private constructor(options: DefaultCatalogCollatorFactoryOptions) { + private constructor(options: { + locationTemplate: string; + filter: GetEntitiesRequest['filter']; + batchSize: number; + entityTransformer?: CatalogCollatorEntityTransformer; + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + catalogClient?: CatalogApi; + }) { const { batchSize, discovery, @@ -75,10 +110,9 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { entityTransformer, } = options; - this.locationTemplate = - locationTemplate || '/catalog/:namespace/:kind/:name'; + this.locationTemplate = locationTemplate; this.filter = filter; - this.batchSize = batchSize || 500; + this.batchSize = batchSize; this.catalogClient = catalogClient || new CatalogClient({ discoveryApi: discovery }); this.tokenManager = tokenManager; diff --git a/plugins/search-backend-module-catalog/src/collators/config.test.ts b/plugins/search-backend-module-catalog/src/collators/config.test.ts new file mode 100644 index 0000000000..dc330f6df2 --- /dev/null +++ b/plugins/search-backend-module-catalog/src/collators/config.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { + defaults, + readCollatorConfigOptions, + readScheduleConfigOptions, +} from './config'; + +describe('config', () => { + describe('readScheduleConfigOptions', () => { + it('reads config', () => { + const result = readScheduleConfigOptions( + new ConfigReader({ + search: { + collators: { + catalog: { + schedule: { + frequency: { minutes: 1 }, + timeout: { seconds: 2 }, + initialDelay: { hours: 3 }, + }, + }, + }, + }, + }), + ); + expect(result).toEqual({ + frequency: { minutes: 1 }, + timeout: { seconds: 2 }, + initialDelay: { hours: 3 }, + }); + }); + + it('returns defaults', () => { + const result = readScheduleConfigOptions(new ConfigReader({})); + expect(result).toEqual(defaults.schedule); + }); + }); + + describe('readCollatorConfigOptions', () => { + it('reads config', () => { + const result = readCollatorConfigOptions( + new ConfigReader({ + search: { + collators: { + catalog: { + batchSize: 1, + locationTemplate: '/mock/:namespace/:kind/:name', + filter: { a: 1 }, + }, + }, + }, + }), + ); + expect(result).toEqual({ + batchSize: 1, + locationTemplate: '/mock/:namespace/:kind/:name', + filter: { a: 1 }, + }); + }); + + it('returns defaults', () => { + const result = readCollatorConfigOptions(new ConfigReader({})); + expect(result).toEqual(defaults.collatorOptions); + }); + }); +}); diff --git a/plugins/search-backend-module-catalog/src/collators/config.ts b/plugins/search-backend-module-catalog/src/collators/config.ts new file mode 100644 index 0000000000..d4ae9b5934 --- /dev/null +++ b/plugins/search-backend-module-catalog/src/collators/config.ts @@ -0,0 +1,81 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + readTaskScheduleDefinitionFromConfig, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { EntityFilterQuery } from '@backstage/catalog-client'; +import { Config } from '@backstage/config'; +import { InputError } from '@backstage/errors'; + +const configKey = 'search.collators.catalog'; + +export const defaults = { + schedule: { + frequency: { minutes: 10 }, + timeout: { minutes: 15 }, + initialDelay: { seconds: 3 }, + }, + collatorOptions: { + locationTemplate: '/catalog/:namespace/:kind/:name', + filter: undefined, + batchSize: 500, + }, +}; + +export function readScheduleConfigOptions( + configRoot: Config, +): TaskScheduleDefinition { + let schedule: TaskScheduleDefinition | undefined = undefined; + + const config = configRoot.getOptionalConfig(configKey); + if (config) { + const scheduleConfig = config.getOptionalConfig('schedule'); + if (scheduleConfig) { + try { + schedule = readTaskScheduleDefinitionFromConfig(scheduleConfig); + } catch (error) { + throw new InputError(`Invalid schedule at ${configKey}, ${error}`); + } + } + } + + return schedule ?? defaults.schedule; +} + +export function readCollatorConfigOptions(configRoot: Config): { + locationTemplate: string; + filter: EntityFilterQuery | undefined; + batchSize: number; +} { + const config = configRoot.getOptionalConfig(configKey); + if (!config) { + return defaults.collatorOptions; + } + + return { + locationTemplate: + config.getOptionalString('locationTemplate') ?? + defaults.collatorOptions.locationTemplate, + filter: + config.getOptionalConfig('filter')?.get() ?? + defaults.collatorOptions.filter, + batchSize: + config.getOptionalNumber('batchSize') ?? + defaults.collatorOptions.batchSize, + }; +} diff --git a/yarn.lock b/yarn.lock index 874da91a9a..d9340b6517 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8444,6 +8444,7 @@ __metadata: "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-permission-common": "workspace:^"