From 84b2516fbb4c00b70b4ab8f42ea509bf466a773a Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Sat, 29 Jun 2024 11:18:27 -0400 Subject: [PATCH 1/4] fix(docs): add scheduler to custom provider example Signed-off-by: aramissennyeydd --- .../software-catalog/external-integrations.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 6eaee2f80d..c77ed51943 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -300,9 +300,19 @@ export const catalogModuleFrobsProvider = createBackendModule({ deps: { catalog: catalogProcessingExtensionPoint, reader: coreServices.urlReader, + scheduler: coreServices.scheduler, }, - async init({ catalog, reader }) { - catalog.addEntityProvider(new FrobsProvider('dev', reader)); + async init({ catalog, reader, scheduler }) { + const frobs = new FrobsProvider('dev', reader); + catalog.addEntityProvider(frobs); + await scheduler.scheduleTask({ + id: 'run_frobs_refresh', + fn: async () => { + await frobs.run(); + }, + frequency: { minutes: 30 }, + timeout: { minutes: 10 }, + }); }, }); }, From 3b118003a93d9f4954b45386e52c05870195a5c9 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Tue, 2 Jul 2024 09:23:33 -0400 Subject: [PATCH 2/4] update to use taskRunner and show how to use config Signed-off-by: aramissennyeydd --- .../software-catalog/external-integrations.md | 106 ++++++++++++++---- 1 file changed, 85 insertions(+), 21 deletions(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index c77ed51943..c269030f29 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -84,6 +84,7 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; +import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api'; /** * Provides entities from fictional frobs service. @@ -92,11 +93,17 @@ export class FrobsProvider implements EntityProvider { private readonly env: string; private readonly reader: UrlReader; private connection?: EntityProviderConnection; + private taskRunner: SchedulerServiceTaskRunner; /** [1] */ - constructor(env: string, reader: UrlReader) { + constructor( + env: string, + reader: UrlReader, + taskRunner: SchedulerServiceTaskRunner, + ) { this.env = env; this.reader = reader; + this.taskRunner = taskRunner; } /** [2] */ @@ -107,6 +114,12 @@ export class FrobsProvider implements EntityProvider { /** [3] */ async connect(connection: EntityProviderConnection): Promise { this.connection = connection; + this.taskRunner.run({ + id: this.getProviderName(), + fn: async () => { + await this.run(); + }, + }); } /** [4] */ @@ -248,24 +261,17 @@ export default async function createPlugin( ): Promise { const builder = CatalogBuilder.create(env); /* highlight-add-start */ - const frobs = new FrobsProvider('production', env.reader); + const taskRunner = env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 10 }, + }); + const frobs = new FrobsProvider('production', env.reader, taskRunner); builder.addEntityProvider(frobs); /* highlight-add-end */ const { processingEngine, router } = await builder.build(); await processingEngine.start(); - /* highlight-add-start */ - await env.scheduler.scheduleTask({ - id: 'run_frobs_refresh', - fn: async () => { - await frobs.run(); - }, - frequency: { minutes: 30 }, - timeout: { minutes: 10 }, - }); - /* highlight-add-end */ - // .. } ``` @@ -300,19 +306,17 @@ export const catalogModuleFrobsProvider = createBackendModule({ deps: { catalog: catalogProcessingExtensionPoint, reader: coreServices.urlReader, + /* highlight-add-start */ scheduler: coreServices.scheduler, + /* highlight-add-end */ }, - async init({ catalog, reader, scheduler }) { - const frobs = new FrobsProvider('dev', reader); - catalog.addEntityProvider(frobs); - await scheduler.scheduleTask({ - id: 'run_frobs_refresh', - fn: async () => { - await frobs.run(); - }, + async init({ catalog, reader, scheduler, config }) { + const taskRunner = scheduler.createScheduledTaskRunner({ frequency: { minutes: 30 }, timeout: { minutes: 10 }, }); + const frobs = new FrobsProvider('dev', reader, taskRunner); + catalog.addEntityProvider(frobs); }, }); }, @@ -328,6 +332,66 @@ backend.add(catalogModuleFrobsProvider); backend.start(); ``` +#### Follow-up: Config Defined Schedule + +If you want to go a step further and increase the configurability of your new `FrobsProvider`, you can define the schedule that the task runs at in `app-config.yaml` instead of requiring code changes to adjust. + +```yaml +catalog: + providers: + frobs-provider: + schedule: + initialDelay: { seconds: 30 } + frequency: { hours: 1 } + timeout: { minutes: 50 } +``` + +This approach will also allow you to customize the schedule per environment + +#### New Backend + +```ts +import { + SchedulerServiceTaskScheduleDefinition, + /* highlight-add-start */ + readSchedulerServiceTaskScheduleDefinitionFromConfig, + /* highlight-add-end */ +} from '@backstage/backend-plugin-api'; + +export const catalogModuleFrobsProvider = createBackendModule({ + pluginId: 'catalog', + moduleId: 'frobs-provider', + register(env) { + env.registerInit({ + deps: { + // ... other deps + /* highlight-add-start */ + rootConfig: coreServices.rootConfig, + /* highlight-add-end */ + }, + async init({ catalog, reader, scheduler, rootConfig }) { + /* highlight-add-start */ + const config = rootConfig.getConfig('catalog.providers.frobs-provider'); // Generally, catalog config goes under catalog.providers.pluginId + // Add a default schedule if you don't define one in config. + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : { + frequency: { minutes: 30 }, + timeout: { minutes: 10 }, + }; + const taskRunner: SchedulerServiceTaskRunner = + scheduler.createScheduledTaskRunner(schedule); + /* highlight-add-end */ + + // rest of your code + }, + }); + }, +}); +``` + ### Example User Entity Provider If you have a 3rd party entity provider such as an internal HR system that you wish to use you are not limited to using our entity providers, (or simply wish to add to existing entity providers with your own data). From fc221e0e57a52d52398140a5d7e4e93eb089e154 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> Date: Tue, 2 Jul 2024 09:24:29 -0400 Subject: [PATCH 3/4] Update docs/features/software-catalog/external-integrations.md Signed-off-by: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> --- docs/features/software-catalog/external-integrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index c269030f29..c9fed03e23 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -310,7 +310,7 @@ export const catalogModuleFrobsProvider = createBackendModule({ scheduler: coreServices.scheduler, /* highlight-add-end */ }, - async init({ catalog, reader, scheduler, config }) { + async init({ catalog, reader, scheduler }) { const taskRunner = scheduler.createScheduledTaskRunner({ frequency: { minutes: 30 }, timeout: { minutes: 10 }, From 82d14dd868f6701673f56e80a8a88dcda21945c4 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Tue, 2 Jul 2024 09:29:01 -0400 Subject: [PATCH 4/4] show how to do it for the old backend as well Signed-off-by: aramissennyeydd --- .../software-catalog/external-integrations.md | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index c9fed03e23..56423c9392 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -336,7 +336,7 @@ backend.start(); If you want to go a step further and increase the configurability of your new `FrobsProvider`, you can define the schedule that the task runs at in `app-config.yaml` instead of requiring code changes to adjust. -```yaml +```yaml title="app-config.yaml" catalog: providers: frobs-provider: @@ -350,7 +350,7 @@ This approach will also allow you to customize the schedule per environment #### New Backend -```ts +```ts title="packages/backend/src/index.ts" import { SchedulerServiceTaskScheduleDefinition, /* highlight-add-start */ @@ -392,6 +392,38 @@ export const catalogModuleFrobsProvider = createBackendModule({ }); ``` +#### Old Backend + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { FrobsProvider } from '../path/to/class'; +import { + /* highlight-add-start */ + readSchedulerServiceTaskScheduleDefinitionFromConfig, + /* highlight-add-end */ +} from '@backstage/backend-plugin-api'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + /* highlight-add-start */ + const config = env.config.getConfig('catalog.providers.frobs-provider'); // Generally, catalog config goes under catalog.providers.pluginId + // Add a default schedule if you don't define one in config. + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : { + frequency: { minutes: 30 }, + timeout: { minutes: 10 }, + }; + const taskRunner = env.scheduler.createScheduledTaskRunner(schedule); + /* highlight-add-end */ + + // .. +} +``` + ### Example User Entity Provider If you have a 3rd party entity provider such as an internal HR system that you wish to use you are not limited to using our entity providers, (or simply wish to add to existing entity providers with your own data).