diff --git a/.changeset/clean-planets-rhyme.md b/.changeset/clean-planets-rhyme.md new file mode 100644 index 0000000000..a6878b491b --- /dev/null +++ b/.changeset/clean-planets-rhyme.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-azure': patch +--- + +`AzureDevOpsEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/azure/discovery diff --git a/docs/integrations/azure/discovery.md b/docs/integrations/azure/discovery.md index 6a420462c0..2156f60814 100644 --- a/docs/integrations/azure/discovery.md +++ b/docs/integrations/azure/discovery.md @@ -42,12 +42,17 @@ catalog: project: myproject repository: service-* # this will match all repos starting with service-* path: /catalog-info.yaml + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } anotherProviderId: # another identifier organization: myorg project: myproject repository: '*' # this will match all repos path: /src/*/catalog-info.yaml # this will search for files deep inside the /src folder - yetAotherProviderId: # guess, what? Another one :) + yetAnotherProviderId: # guess, what? Another one :) host: selfhostedazure.yourcompany.com organization: myorg project: myproject @@ -55,11 +60,20 @@ catalog: The parameters available are: -- `host:` Leave empty for Cloud hosted, otherwise set to your self-hosted instance host. -- `organization:` Your Organization slug (or Collection for on-premise users). Required. -- `project:` Your project slug. Required. -- `repository:` The repository name. Wildcards are supported as show on the examples above. If not set, all repositories will be searched. -- `path:` Where to find catalog-info.yaml files. Defaults to /catalog-info.yaml. +- **`host:`** _(optional)_ Leave empty for Cloud hosted, otherwise set to your self-hosted instance host. +- **`organization:`** Your Organization slug (or Collection for on-premise users). Required. +- **`project:`** Your project slug. Required. +- **`repository:`** _(optional)_ The repository name. Wildcards are supported as show on the examples above. If not set, all repositories will be searched. +- **`path:`** _(optional)_ Where to find catalog-info.yaml files. Defaults to /catalog-info.yaml. +- **`schedule`** _(optional)_: + - **`frequency`**: + How often you want the task to run. The system does its best to avoid overlapping invocations. + - **`timeout`**: + The maximum amount of time that a single task invocation can take. + - **`initialDelay`** _(optional)_: + The amount of time that should pass before the first invocation happens. + - **`scope`** _(optional)_: + `'global'` or `'local'`. Sets the scope of concurrency control. _Note:_ the path parameter follows the same rules as the search on Azure DevOps web interface. For more details visit the @@ -84,10 +98,13 @@ const builder = await CatalogBuilder.create(env); +builder.addEntityProvider( + AzureDevOpsEntityProvider.fromConfig(env.config, { + logger: env.logger, ++ // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ -+ frequency: Duration.fromObject({ minutes: 30 }), -+ timeout: Duration.fromObject({ minutes: 3 }), ++ frequency: { minutes: 30 }, ++ timeout: { minutes: 3 }, + }), ++ // optional: alternatively, use schedule ++ scheduler: env.scheduler, + }), +); ``` diff --git a/plugins/catalog-backend-module-azure/api-report.md b/plugins/catalog-backend-module-azure/api-report.md index 22f5015c9b..8fb7edeb53 100644 --- a/plugins/catalog-backend-module-azure/api-report.md +++ b/plugins/catalog-backend-module-azure/api-report.md @@ -10,6 +10,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { TaskRunner } from '@backstage/backend-tasks'; @@ -45,7 +46,8 @@ export class AzureDevOpsEntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): AzureDevOpsEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-azure/config.d.ts b/plugins/catalog-backend-module-azure/config.d.ts index 6e423c9415..940695c8ea 100644 --- a/plugins/catalog-backend-module-azure/config.d.ts +++ b/plugins/catalog-backend-module-azure/config.d.ts @@ -14,34 +14,35 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + interface AzureDevOpsConfig { /** * (Optional) The DevOps host; leave empty for `dev.azure.com`, otherwise set to your self-hosted instance host. - * @visibility backend */ host: string; /** * (Required) Your organization slug. - * @visibility backend */ organization: string; /** * (Required) Your project slug. - * @visibility backend */ project: string; /** * (Optional) The repository name. Wildcards are supported as show on the examples above. * If not set, all repositories will be searched. - * @visibility backend */ repository?: string; /** * (Optional) Where to find catalog-info.yaml files. Wildcards are supported. * If not set, defaults to /catalog-info.yaml. - * @visibility backend */ path?: string; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } export interface Config { diff --git a/plugins/catalog-backend-module-azure/package.json b/plugins/catalog-backend-module-azure/package.json index 0385655259..00dbc0d744 100644 --- a/plugins/catalog-backend-module-azure/package.json +++ b/plugins/catalog-backend-module-azure/package.json @@ -49,7 +49,8 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", - "@types/lodash": "^4.14.151" + "@types/lodash": "^4.14.151", + "luxon": "^3.0.0" }, "files": [ "dist", diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts index c57455d8d6..f7a63f259e 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts @@ -15,7 +15,11 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { CodeSearchResultItem } from '../lib'; @@ -52,6 +56,7 @@ describe('AzureDevOpsEntityProvider', () => { expectedBaseUrl: string, names: Record, integrationConfig?: object, + scheduleInConfig?: boolean, ) => { const config = new ConfigReader({ integrations: { @@ -74,9 +79,18 @@ describe('AzureDevOpsEntityProvider', () => { refresh: jest.fn(), }; + const schedulingConfig: Record = {}; + if (scheduleInConfig) { + schedulingConfig.scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + } else { + schedulingConfig.schedule = schedule; + } + const provider = AzureDevOpsEntityProvider.fromConfig(config, { + ...schedulingConfig, logger, - schedule, })[0]; expect(provider.getProviderName()).toEqual( `AzureDevOpsEntityProvider:${providerId}`, @@ -193,4 +207,92 @@ describe('AzureDevOpsEntityProvider', () => { }, ); }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + azureDevOps: { + test: { + organization: 'myorganization', + project: 'myproject', + }, + }, + }, + }, + }); + + expect(() => + AzureDevOpsEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + azureDevOps: { + test: { + organization: 'myorganization', + project: 'myproject', + }, + }, + }, + }, + }); + + expect(() => + AzureDevOpsEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for AzureDevOpsEntityProvider:test', + ); + }); + + // eslint-disable-next-line jest/expect-expect + it('single simple provider config with schedule in config', async () => { + return expectMutation( + 'allReposMultipleFiles', + { + organization: 'myorganization', + project: 'myproject', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'myrepo', + }, + }, + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'myotherrepo', + }, + }, + ], + 'https://dev.azure.com/myorganization/myproject', + { + 'myrepo?path=/catalog-info.yaml': + 'generated-87865246726bb12a8c4fb4f914443f1fbb91648c', + 'myotherrepo?path=/catalog-info.yaml': + 'generated-2deccac384c34d0dca37be0ebb4b1c8cf6913fe1', + }, + ); + }); }); diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts index aca30f367a..b824d176ee 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { AzureIntegration, ScmIntegrations } from '@backstage/integration'; import { @@ -45,11 +45,16 @@ export class AzureDevOpsEntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): AzureDevOpsEntityProvider[] { const providerConfigs = readAzureDevOpsConfigs(configRoot); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + return providerConfigs.map(providerConfig => { const integration = ScmIntegrations.fromConfig(configRoot).azure.byHost( providerConfig.host, @@ -61,11 +66,21 @@ export class AzureDevOpsEntityProvider implements EntityProvider { ); } + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for AzureDevOpsEntityProvider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + return new AzureDevOpsEntityProvider( providerConfig, integration, options.logger, - options.schedule, + taskRunner, ); }); } @@ -74,19 +89,19 @@ export class AzureDevOpsEntityProvider implements EntityProvider { private readonly config: AzureDevOpsConfig, private readonly integration: AzureIntegration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, ) { this.logger = logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-azure/src/providers/config.test.ts b/plugins/catalog-backend-module-azure/src/providers/config.test.ts index 236c9a48d4..a09079f1c3 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readAzureDevOpsConfigs } from './config'; describe('readAzureDevOpsConfigs', () => { @@ -33,17 +34,27 @@ describe('readAzureDevOpsConfigs', () => { project: 'myproject', repository: 'service-*', }; + const provider4 = { + organization: 'mycompany', + project: 'myproject', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }; const config = { catalog: { providers: { - azureDevOps: { provider1, provider2, provider3 }, + azureDevOps: { provider1, provider2, provider3, provider4 }, }, }, }; const actual = readAzureDevOpsConfigs(new ConfigReader(config)); - expect(actual).toHaveLength(3); + expect(actual).toHaveLength(4); expect(actual[0]).toEqual({ ...provider1, path: '/catalog-info.yaml', @@ -63,5 +74,16 @@ describe('readAzureDevOpsConfigs', () => { path: '/catalog-info.yaml', id: 'provider3', }); + expect(actual[3]).toEqual({ + ...provider4, + host: 'dev.azure.com', + path: '/catalog-info.yaml', + repository: '*', + id: 'provider4', + schedule: { + ...provider4.schedule, + frequency: Duration.fromISO(provider4.schedule.frequency), + }, + }); }); }); diff --git a/plugins/catalog-backend-module-azure/src/providers/config.ts b/plugins/catalog-backend-module-azure/src/providers/config.ts index 1d143fc882..c84c242488 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { AzureDevOpsConfig } from './types'; @@ -42,6 +43,10 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig { const repository = config.getOptionalString('repository') || '*'; const path = config.getOptionalString('path') || '/catalog-info.yaml'; + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, host, @@ -49,5 +54,6 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig { project, repository, path, + schedule, }; } diff --git a/plugins/catalog-backend-module-azure/src/providers/types.ts b/plugins/catalog-backend-module-azure/src/providers/types.ts index 15ea00ff13..b22a8551a5 100644 --- a/plugins/catalog-backend-module-azure/src/providers/types.ts +++ b/plugins/catalog-backend-module-azure/src/providers/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + export type AzureDevOpsConfig = { id: string; host: string; @@ -21,4 +23,5 @@ export type AzureDevOpsConfig = { project: string; repository: string; path: string; + schedule?: TaskScheduleDefinition; }; diff --git a/yarn.lock b/yarn.lock index 2bd6f7bfdd..23830a549a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4477,6 +4477,7 @@ __metadata: "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 lodash: ^4.17.21 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 uuid: ^8.0.0