paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 48 Packages Projects Releases Wiki Activity

Merge pull request #19278 from backstage/freben/module-config

Browse Source 
make `searchModuleCatalogCollator` configurable from app-config
This commit is contained in:
Fredrik Adelöw
2023-08-14 15:14:40 +02:00
committed by GitHub
parent a796cc04e7 3c74b89896
commit 9f28cb2b50
12 changed files with 394 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/metal-kings-argue.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-search-backend-module-catalog': patch
---
Breaking change in the alpha export `searchModuleCatalogCollator`: Moved collator settings from module options into app-config. You are now expected to set up the catalog collator under the `search.collators.catalog` configuration key. There is also a new `catalogCollatorExtensionPoint` extension point for the module, wherein you can set custom transformers.
plugins/search-backend-module-catalog/alpha-api-report.md
+10 -12
View File
@@ -4,21 +4,19 @@
```ts
import { BackendFeature } from '@backstage/backend-plugin-api';
import { DefaultCatalogCollatorFactoryOptions } from '@backstage/plugin-search-backend-module-catalog';
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
import { CatalogCollatorEntityTransformer } from '@backstage/plugin-search-backend-module-catalog';
import { ExtensionPoint } from '@backstage/backend-plugin-api';
// @alpha
export const searchModuleCatalogCollator: (
options?: SearchModuleCatalogCollatorOptions | undefined,
) => BackendFeature;
// @alpha
export type SearchModuleCatalogCollatorOptions = Omit<
DefaultCatalogCollatorFactoryOptions,
'discovery' | 'tokenManager' | 'catalogClient'
> & {
schedule?: TaskScheduleDefinition;
export type CatalogCollatorExtensionPoint = {
setEntityTransformer(transformer: CatalogCollatorEntityTransformer): void;
};
// @alpha
export const catalogCollatorExtensionPoint: ExtensionPoint<CatalogCollatorExtensionPoint>;
// @alpha
export const searchModuleCatalogCollator: () => BackendFeature;
// (No @packageDocumentation comment for this package)
```
plugins/search-backend-module-catalog/api-report.md
+2 -2
View File
@@ -24,11 +24,11 @@ export type CatalogCollatorEntityTransformer = (
// @public (undocumented)
export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer;
// @public (undocumented)
// @public
export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
// (undocumented)
static fromConfig(
_config: Config,
configRoot: Config,
options: DefaultCatalogCollatorFactoryOptions,
): DefaultCatalogCollatorFactory;
// (undocumented)
plugins/search-backend-module-catalog/config.d.ts
Vendored
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks';
export interface Config {
search?: {
collators?: {
/**
* Configuration options for `@backstage/plugin-search-backend-module-catalog`
*/
catalog?: {
/**
* A templating string with placeholders, to form the final location of
* the entity.
*
* Defaults to '/catalog/:namespace/:kind/:name'
*/
locationTemplate?: string;
/**
* A filter expression passed to the catalog client, to select what
* entities to collate.
*
* Defaults to no filter, ie indexing all entities.
*/
filter?: object;
/**
* The number of entities to process at a time. Keep this at a
* reasonable number to avoid overloading either the catalog or the
* search backend.
*
* Defaults to 500
*/
batchSize?: number;
/**
* The schedule for how often to run the collation job.
*/
schedule?: TaskScheduleDefinitionConfig;
};
};
};
}
plugins/search-backend-module-catalog/package.json
+5 -2
View File
@@ -48,6 +48,7 @@
"@backstage/catalog-client": "workspace:^",
"@backstage/catalog-model": "workspace:^",
"@backstage/config": "workspace:^",
"@backstage/errors": "workspace:^",
"@backstage/plugin-catalog-common": "workspace:^",
"@backstage/plugin-catalog-node": "workspace:^",
"@backstage/plugin-permission-common": "workspace:^",
@@ -61,6 +62,8 @@
"msw": "^1.0.0"
},
"files": [
"dist"
]
"dist",
"config.d.ts"
],
"configSchema": "config.d.ts"
}
plugins/search-backend-module-catalog/src/alpha.test.ts
+35 -8
View File
@@ -14,17 +14,11 @@
* limitations under the License.
*/
import { startTestBackend } from '@backstage/backend-test-utils';
import { mockServices, startTestBackend } from '@backstage/backend-test-utils';
import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha';
import { searchModuleCatalogCollator } from './alpha';
describe('searchModuleCatalogCollator', () => {
const schedule = {
frequency: { minutes: 10 },
timeout: { minutes: 15 },
initialDelay: { seconds: 3 },
};
it('should register the catalog collator to the search index registry extension point with factory and schedule', async () => {
const extensionPointMock = {
addCollator: jest.fn(),
@@ -34,7 +28,7 @@ describe('searchModuleCatalogCollator', () => {
extensionPoints: [
[searchIndexRegistryExtensionPoint, extensionPointMock],
],
features: [searchModuleCatalogCollator({ schedule })],
features: [searchModuleCatalogCollator()],
});
expect(extensionPointMock.addCollator).toHaveBeenCalledTimes(1);
@@ -43,4 +37,37 @@ describe('searchModuleCatalogCollator', () => {
schedule: expect.objectContaining({ run: expect.any(Function) }),
});
});
it('refuses to start up with a broken schedule', async () => {
await expect(
startTestBackend({
extensionPoints: [
[
searchIndexRegistryExtensionPoint,
{
addCollator: jest.fn(),
},
],
],
features: [searchModuleCatalogCollator()],
services: [
mockServices.rootConfig.factory({
data: {
search: {
collators: {
catalog: {
schedule: {
frequency: { minutes: 10 },
timeout: { minutes: 'boo' },
initialDelay: { seconds: 3 },
},
},
},
},
},
}),
],
}),
).rejects.toThrow(/Invalid schedule/);
});
});
plugins/search-backend-module-catalog/src/alpha.ts
+72 -56
View File
@@ -22,73 +22,89 @@
import {
coreServices,
createBackendModule,
createExtensionPoint,
} from '@backstage/backend-plugin-api';
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha';
import {
DefaultCatalogCollatorFactory,
DefaultCatalogCollatorFactoryOptions,
} from '@backstage/plugin-search-backend-module-catalog';
import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha';
import {
CatalogCollatorEntityTransformer,
DefaultCatalogCollatorFactory,
} from '@backstage/plugin-search-backend-module-catalog';
import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha';
import { readScheduleConfigOptions } from './collators/config';
/**
* Options for {@link catalogCollatorExtensionPoint}.
*
* @alpha
* Options for {@link searchModuleCatalogCollator}.
*/
export type SearchModuleCatalogCollatorOptions = Omit<
DefaultCatalogCollatorFactoryOptions,
'discovery' | 'tokenManager' | 'catalogClient'
> & {
schedule?: TaskScheduleDefinition;
export type CatalogCollatorExtensionPoint = {
/**
* Allows you to customize how entities are shaped into documents.
*/
setEntityTransformer(transformer: CatalogCollatorEntityTransformer): void;
};
/**
* Extension point for customizing how catalog entities are shaped into
* documents for the search backend, when using
* {@link searchModuleCatalogCollator}.
*
* @alpha
*/
export const catalogCollatorExtensionPoint =
createExtensionPoint<CatalogCollatorExtensionPoint>({
id: 'search.catalogCollator.extension',
});
/**
* Search backend module for the Catalog index.
*
* @alpha
*/
export const searchModuleCatalogCollator = createBackendModule(
(options?: SearchModuleCatalogCollatorOptions) => ({
moduleId: 'catalogCollator',
pluginId: 'search',
register(env) {
env.registerInit({
deps: {
config: coreServices.rootConfig,
discovery: coreServices.discovery,
tokenManager: coreServices.tokenManager,
scheduler: coreServices.scheduler,
indexRegistry: searchIndexRegistryExtensionPoint,
catalog: catalogServiceRef,
},
async init({
config,
discovery,
tokenManager,
scheduler,
indexRegistry,
catalog,
}) {
const defaultSchedule = {
frequency: { minutes: 10 },
timeout: { minutes: 15 },
initialDelay: { seconds: 3 },
};
export const searchModuleCatalogCollator = createBackendModule({
moduleId: 'catalogCollator',
pluginId: 'search',
register(env) {
let entityTransformer: CatalogCollatorEntityTransformer | undefined;
indexRegistry.addCollator({
schedule: scheduler.createScheduledTaskRunner(
options?.schedule ?? defaultSchedule,
),
factory: DefaultCatalogCollatorFactory.fromConfig(config, {
...options,
discovery,
tokenManager,
catalogClient: catalog,
}),
});
},
});
},
}),
);
env.registerExtensionPoint(catalogCollatorExtensionPoint, {
setEntityTransformer(transformer) {
if (entityTransformer) {
throw new Error('setEntityTransformer can only be called once');
}
entityTransformer = transformer;
},
});
env.registerInit({
deps: {
config: coreServices.rootConfig,
discovery: coreServices.discovery,
tokenManager: coreServices.tokenManager,
scheduler: coreServices.scheduler,
indexRegistry: searchIndexRegistryExtensionPoint,
catalog: catalogServiceRef,
},
async init({
config,
discovery,
tokenManager,
scheduler,
indexRegistry,
catalog,
}) {
indexRegistry.addCollator({
schedule: scheduler.createScheduledTaskRunner(
readScheduleConfigOptions(config),
),
factory: DefaultCatalogCollatorFactory.fromConfig(config, {
entityTransformer,
discovery,
tokenManager,
catalogClient: catalog,
}),
});
},
});
},
});
plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.test.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
PluginEndpointDiscovery,
TokenManager,
plugins/search-backend-module-catalog/src/collators/DefaultCatalogCollatorFactory.ts
+45 -11
View File
@@ -14,7 +14,6 @@
* limitations under the License.
*/
import { Config } from '@backstage/config';
import {
PluginEndpointDiscovery,
TokenManager,
@@ -24,27 +23,45 @@ import {
CatalogClient,
GetEntitiesRequest,
} from '@backstage/catalog-client';
import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common/alpha';
import { stringifyEntityRef } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { CatalogEntityDocument } from '@backstage/plugin-catalog-common';
import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common/alpha';
import { Permission } from '@backstage/plugin-permission-common';
import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
import { Readable } from 'stream';
import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
import { readCollatorConfigOptions } from './config';
import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer';
import { stringifyEntityRef } from '@backstage/catalog-model';
/** @public */
export type DefaultCatalogCollatorFactoryOptions = {
discovery: PluginEndpointDiscovery;
tokenManager: TokenManager;
/**
* @deprecated Use the config key `search.collators.catalog.locationTemplate` instead.
*/
locationTemplate?: string;
/**
* @deprecated Use the config key `search.collators.catalog.filter` instead.
*/
filter?: GetEntitiesRequest['filter'];
/**
* @deprecated Use the config key `search.collators.catalog.batchSize` instead.
*/
batchSize?: number;
catalogClient?: CatalogApi;
/**
* Allows you to customize how entities are shaped into documents.
*/
entityTransformer?: CatalogCollatorEntityTransformer;
};
/** @public */
/**
* Collates entities from the Catalog into documents for the search backend.
*
* @public
*/
export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
public readonly type = 'software-catalog';
public readonly visibilityPermission: Permission =
@@ -58,13 +75,31 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
private entityTransformer: CatalogCollatorEntityTransformer;
static fromConfig(
_config: Config,
configRoot: Config,
options: DefaultCatalogCollatorFactoryOptions,
) {
return new DefaultCatalogCollatorFactory(options);
const configOptions = readCollatorConfigOptions(configRoot);
return new DefaultCatalogCollatorFactory({
locationTemplate:
options.locationTemplate ?? configOptions.locationTemplate,
filter: options.filter ?? configOptions.filter,
batchSize: options.batchSize ?? configOptions.batchSize,
entityTransformer: options.entityTransformer,
discovery: options.discovery,
tokenManager: options.tokenManager,
catalogClient: options.catalogClient,
});
}
private constructor(options: DefaultCatalogCollatorFactoryOptions) {
private constructor(options: {
locationTemplate: string;
filter: GetEntitiesRequest['filter'];
batchSize: number;
entityTransformer?: CatalogCollatorEntityTransformer;
discovery: PluginEndpointDiscovery;
tokenManager: TokenManager;
catalogClient?: CatalogApi;
}) {
const {
batchSize,
discovery,
@@ -75,10 +110,9 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
entityTransformer,
} = options;
this.locationTemplate =
locationTemplate || '/catalog/:namespace/:kind/:name';
this.locationTemplate = locationTemplate;
this.filter = filter;
this.batchSize = batchSize || 500;
this.batchSize = batchSize;
this.catalogClient =
catalogClient || new CatalogClient({ discoveryApi: discovery });
this.tokenManager = tokenManager;
plugins/search-backend-module-catalog/src/collators/config.test.ts
+82
View File
@@ -0,0 +1,82 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import {
defaults,
readCollatorConfigOptions,
readScheduleConfigOptions,
} from './config';
describe('config', () => {
describe('readScheduleConfigOptions', () => {
it('reads config', () => {
const result = readScheduleConfigOptions(
new ConfigReader({
search: {
collators: {
catalog: {
schedule: {
frequency: { minutes: 1 },
timeout: { seconds: 2 },
initialDelay: { hours: 3 },
},
},
},
},
}),
);
expect(result).toEqual({
frequency: { minutes: 1 },
timeout: { seconds: 2 },
initialDelay: { hours: 3 },
});
});
it('returns defaults', () => {
const result = readScheduleConfigOptions(new ConfigReader({}));
expect(result).toEqual(defaults.schedule);
});
});
describe('readCollatorConfigOptions', () => {
it('reads config', () => {
const result = readCollatorConfigOptions(
new ConfigReader({
search: {
collators: {
catalog: {
batchSize: 1,
locationTemplate: '/mock/:namespace/:kind/:name',
filter: { a: 1 },
},
},
},
}),
);
expect(result).toEqual({
batchSize: 1,
locationTemplate: '/mock/:namespace/:kind/:name',
filter: { a: 1 },
});
});
it('returns defaults', () => {
const result = readCollatorConfigOptions(new ConfigReader({}));
expect(result).toEqual(defaults.collatorOptions);
});
});
});
plugins/search-backend-module-catalog/src/collators/config.ts
+81
View File
@@ -0,0 +1,81 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
readTaskScheduleDefinitionFromConfig,
TaskScheduleDefinition,
} from '@backstage/backend-tasks';
import { EntityFilterQuery } from '@backstage/catalog-client';
import { Config } from '@backstage/config';
import { InputError } from '@backstage/errors';
const configKey = 'search.collators.catalog';
export const defaults = {
schedule: {
frequency: { minutes: 10 },
timeout: { minutes: 15 },
initialDelay: { seconds: 3 },
},
collatorOptions: {
locationTemplate: '/catalog/:namespace/:kind/:name',
filter: undefined,
batchSize: 500,
},
};
export function readScheduleConfigOptions(
configRoot: Config,
): TaskScheduleDefinition {
let schedule: TaskScheduleDefinition | undefined = undefined;
const config = configRoot.getOptionalConfig(configKey);
if (config) {
const scheduleConfig = config.getOptionalConfig('schedule');
if (scheduleConfig) {
try {
schedule = readTaskScheduleDefinitionFromConfig(scheduleConfig);
} catch (error) {
throw new InputError(`Invalid schedule at ${configKey}, ${error}`);
}
}
}
return schedule ?? defaults.schedule;
}
export function readCollatorConfigOptions(configRoot: Config): {
locationTemplate: string;
filter: EntityFilterQuery | undefined;
batchSize: number;
} {
const config = configRoot.getOptionalConfig(configKey);
if (!config) {
return defaults.collatorOptions;
}
return {
locationTemplate:
config.getOptionalString('locationTemplate') ??
defaults.collatorOptions.locationTemplate,
filter:
config.getOptionalConfig('filter')?.get<EntityFilterQuery>() ??
defaults.collatorOptions.filter,
batchSize:
config.getOptionalNumber('batchSize') ??
defaults.collatorOptions.batchSize,
};
}
yarn.lock
+1
View File
@@ -8444,6 +8444,7 @@ __metadata:
"@backstage/catalog-model": "workspace:^"
"@backstage/cli": "workspace:^"
"@backstage/config": "workspace:^"
"@backstage/errors": "workspace:^"
"@backstage/plugin-catalog-common": "workspace:^"
"@backstage/plugin-catalog-node": "workspace:^"
"@backstage/plugin-permission-common": "workspace:^"