feat(catalog/azure): Add option to configure schedule via app-config.yaml

Relates-to: PR #13859
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>

.changeset/clean-planets-rhyme.md

@@ -0,0 +1,8 @@
+---
+'@backstage/plugin-catalog-backend-module-azure': patch
+---
+
+`AzureDevOpsEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
+
+Please find how to configure the schedule at the config at
+https://backstage.io/docs/integrations/azure/discovery

docs/integrations/azure/discovery.md

@@ -42,12 +42,17 @@ catalog:
         project: myproject
         repository: service-* # this will match all repos starting with service-*
         path: /catalog-info.yaml
+        schedule: # optional; same options as in TaskScheduleDefinition
+          # supports cron, ISO duration, "human duration" as used in code
+          frequency: { minutes: 30 }
+          # supports ISO duration, "human duration" as used in code
+          timeout: { minutes: 3 }
       anotherProviderId: # another identifier
         organization: myorg
         project: myproject
         repository: '*' # this will match all repos
         path: /src/*/catalog-info.yaml # this will search for files deep inside the /src folder
-      yetAotherProviderId: # guess, what? Another one :)
+      yetAnotherProviderId: # guess, what? Another one :)
         host: selfhostedazure.yourcompany.com
         organization: myorg
         project: myproject
@@ -55,11 +60,20 @@ catalog:
 
 The parameters available are:
 
-- `host:` Leave empty for Cloud hosted, otherwise set to your self-hosted instance host.
-- `organization:` Your Organization slug (or Collection for on-premise users). Required.
-- `project:` Your project slug. Required.
-- `repository:` The repository name. Wildcards are supported as show on the examples above. If not set, all repositories will be searched.
-- `path:` Where to find catalog-info.yaml files. Defaults to /catalog-info.yaml.
+- **`host:`** _(optional)_ Leave empty for Cloud hosted, otherwise set to your self-hosted instance host.
+- **`organization:`** Your Organization slug (or Collection for on-premise users). Required.
+- **`project:`** Your project slug. Required.
+- **`repository:`** _(optional)_ The repository name. Wildcards are supported as show on the examples above. If not set, all repositories will be searched.
+- **`path:`** _(optional)_ Where to find catalog-info.yaml files. Defaults to /catalog-info.yaml.
+- **`schedule`** _(optional)_:
+  - **`frequency`**:
+    How often you want the task to run. The system does its best to avoid overlapping invocations.
+  - **`timeout`**:
+    The maximum amount of time that a single task invocation can take.
+  - **`initialDelay`** _(optional)_:
+    The amount of time that should pass before the first invocation happens.
+  - **`scope`** _(optional)_:
+    `'global'` or `'local'`. Sets the scope of concurrency control.
 
 _Note:_ the path parameter follows the same rules as the search on Azure DevOps
 web interface. For more details visit the
@@ -84,10 +98,13 @@ const builder = await CatalogBuilder.create(env);
 +builder.addEntityProvider(
 +  AzureDevOpsEntityProvider.fromConfig(env.config, {
 +    logger: env.logger,
++    // optional: alternatively, use scheduler with schedule defined in app-config.yaml
 +    schedule: env.scheduler.createScheduledTaskRunner({
-+      frequency: Duration.fromObject({ minutes: 30 }),
-+      timeout: Duration.fromObject({ minutes: 3 }),
++      frequency: { minutes: 30 },
++      timeout: { minutes: 3 },
 +    }),
++    // optional: alternatively, use schedule
++    scheduler: env.scheduler,
 +  }),
 +);
 ```

plugins/catalog-backend-module-azure/api-report.md

@@ -10,6 +10,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
+import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { ScmIntegrationRegistry } from '@backstage/integration';
 import { TaskRunner } from '@backstage/backend-tasks';
 
@@ -45,7 +46,8 @@ export class AzureDevOpsEntityProvider implements EntityProvider {
     configRoot: Config,
     options: {
       logger: Logger;
-      schedule: TaskRunner;
+      schedule?: TaskRunner;
+      scheduler?: PluginTaskScheduler;
     },
   ): AzureDevOpsEntityProvider[];
   // (undocumented)

plugins/catalog-backend-module-azure/config.d.ts

@@ -14,34 +14,35 @@
  * limitations under the License.
  */
 
+import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks';
+
 interface AzureDevOpsConfig {
   /**
    * (Optional) The DevOps host; leave empty for `dev.azure.com`, otherwise set to your self-hosted instance host.
-   * @visibility backend
    */
   host: string;
   /**
    * (Required) Your organization slug.
-   * @visibility backend
    */
   organization: string;
   /**
    * (Required) Your project slug.
-   * @visibility backend
    */
   project: string;
   /**
    * (Optional) The repository name. Wildcards are supported as show on the examples above.
    * If not set, all repositories will be searched.
-   * @visibility backend
    */
   repository?: string;
   /**
    * (Optional) Where to find catalog-info.yaml files. Wildcards are supported.
    * If not set, defaults to /catalog-info.yaml.
-   * @visibility backend
    */
   path?: string;
+  /**
+   * (Optional) TaskScheduleDefinition for the refresh.
+   */
+  schedule?: TaskScheduleDefinitionConfig;
 }
 
 export interface Config {

plugins/catalog-backend-module-azure/package.json

@@ -49,7 +49,8 @@
   "devDependencies": {
     "@backstage/backend-test-utils": "workspace:^",
     "@backstage/cli": "workspace:^",
-    "@types/lodash": "^4.14.151"
+    "@types/lodash": "^4.14.151",
+    "luxon": "^3.0.0"
   },
   "files": [
     "dist",

plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts

@@ -15,7 +15,11 @@
  */
 
 import { getVoidLogger } from '@backstage/backend-common';
-import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
+import {
+  PluginTaskScheduler,
+  TaskInvocationDefinition,
+  TaskRunner,
+} from '@backstage/backend-tasks';
 import { ConfigReader } from '@backstage/config';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { CodeSearchResultItem } from '../lib';
@@ -52,6 +56,7 @@ describe('AzureDevOpsEntityProvider', () => {
     expectedBaseUrl: string,
     names: Record<string, string>,
     integrationConfig?: object,
+    scheduleInConfig?: boolean,
   ) => {
     const config = new ConfigReader({
       integrations: {
@@ -74,9 +79,18 @@ describe('AzureDevOpsEntityProvider', () => {
       refresh: jest.fn(),
     };
 
+    const schedulingConfig: Record<string, any> = {};
+    if (scheduleInConfig) {
+      schedulingConfig.scheduler = {
+        createScheduledTaskRunner: (_: any) => schedule,
+      } as unknown as PluginTaskScheduler;
+    } else {
+      schedulingConfig.schedule = schedule;
+    }
+
     const provider = AzureDevOpsEntityProvider.fromConfig(config, {
+      ...schedulingConfig,
       logger,
-      schedule,
     })[0];
     expect(provider.getProviderName()).toEqual(
       `AzureDevOpsEntityProvider:${providerId}`,
@@ -193,4 +207,92 @@ describe('AzureDevOpsEntityProvider', () => {
       },
     );
   });
+
+  it('fail without schedule and scheduler', () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          azureDevOps: {
+            test: {
+              organization: 'myorganization',
+              project: 'myproject',
+            },
+          },
+        },
+      },
+    });
+
+    expect(() =>
+      AzureDevOpsEntityProvider.fromConfig(config, {
+        logger,
+      }),
+    ).toThrow('Either schedule or scheduler must be provided');
+  });
+
+  it('fail with scheduler but no schedule config', () => {
+    const scheduler = {
+      createScheduledTaskRunner: (_: any) => jest.fn(),
+    } as unknown as PluginTaskScheduler;
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          azureDevOps: {
+            test: {
+              organization: 'myorganization',
+              project: 'myproject',
+            },
+          },
+        },
+      },
+    });
+
+    expect(() =>
+      AzureDevOpsEntityProvider.fromConfig(config, {
+        logger,
+        scheduler,
+      }),
+    ).toThrow(
+      'No schedule provided neither via code nor config for AzureDevOpsEntityProvider:test',
+    );
+  });
+
+  // eslint-disable-next-line jest/expect-expect
+  it('single simple provider config with schedule in config', async () => {
+    return expectMutation(
+      'allReposMultipleFiles',
+      {
+        organization: 'myorganization',
+        project: 'myproject',
+        schedule: {
+          frequency: 'PT30M',
+          timeout: {
+            minutes: 3,
+          },
+        },
+      },
+      [
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/catalog-info.yaml',
+          repository: {
+            name: 'myrepo',
+          },
+        },
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/catalog-info.yaml',
+          repository: {
+            name: 'myotherrepo',
+          },
+        },
+      ],
+      'https://dev.azure.com/myorganization/myproject',
+      {
+        'myrepo?path=/catalog-info.yaml':
+          'generated-87865246726bb12a8c4fb4f914443f1fbb91648c',
+        'myotherrepo?path=/catalog-info.yaml':
+          'generated-2deccac384c34d0dca37be0ebb4b1c8cf6913fe1',
+      },
+    );
+  });
 });

plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { TaskRunner } from '@backstage/backend-tasks';
+import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
 import { Config } from '@backstage/config';
 import { AzureIntegration, ScmIntegrations } from '@backstage/integration';
 import {
@@ -45,11 +45,16 @@ export class AzureDevOpsEntityProvider implements EntityProvider {
     configRoot: Config,
     options: {
       logger: Logger;
-      schedule: TaskRunner;
+      schedule?: TaskRunner;
+      scheduler?: PluginTaskScheduler;
     },
   ): AzureDevOpsEntityProvider[] {
     const providerConfigs = readAzureDevOpsConfigs(configRoot);
 
+    if (!options.schedule && !options.scheduler) {
+      throw new Error('Either schedule or scheduler must be provided.');
+    }
+
     return providerConfigs.map(providerConfig => {
       const integration = ScmIntegrations.fromConfig(configRoot).azure.byHost(
         providerConfig.host,
@@ -61,11 +66,21 @@ export class AzureDevOpsEntityProvider implements EntityProvider {
         );
       }
 
+      if (!options.schedule && !providerConfig.schedule) {
+        throw new Error(
+          `No schedule provided neither via code nor config for AzureDevOpsEntityProvider:${providerConfig.id}.`,
+        );
+      }
+
+      const taskRunner =
+        options.schedule ??
+        options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
+
       return new AzureDevOpsEntityProvider(
         providerConfig,
         integration,
         options.logger,
-        options.schedule,
+        taskRunner,
       );
     });
   }
@@ -74,19 +89,19 @@ export class AzureDevOpsEntityProvider implements EntityProvider {
     private readonly config: AzureDevOpsConfig,
     private readonly integration: AzureIntegration,
     logger: Logger,
-    schedule: TaskRunner,
+    taskRunner: TaskRunner,
   ) {
     this.logger = logger.child({
       target: this.getProviderName(),
     });
 
-    this.scheduleFn = this.createScheduleFn(schedule);
+    this.scheduleFn = this.createScheduleFn(taskRunner);
   }
 
-  private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
+  private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
     return async () => {
       const taskId = `${this.getProviderName()}:refresh`;
-      return schedule.run({
+      return taskRunner.run({
         id: taskId,
         fn: async () => {
           const logger = this.logger.child({

plugins/catalog-backend-module-azure/src/providers/config.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { ConfigReader } from '@backstage/config';
+import { Duration } from 'luxon';
 import { readAzureDevOpsConfigs } from './config';
 
 describe('readAzureDevOpsConfigs', () => {
@@ -33,17 +34,27 @@ describe('readAzureDevOpsConfigs', () => {
       project: 'myproject',
       repository: 'service-*',
     };
+    const provider4 = {
+      organization: 'mycompany',
+      project: 'myproject',
+      schedule: {
+        frequency: 'PT30M',
+        timeout: {
+          minutes: 3,
+        },
+      },
+    };
     const config = {
       catalog: {
         providers: {
-          azureDevOps: { provider1, provider2, provider3 },
+          azureDevOps: { provider1, provider2, provider3, provider4 },
         },
       },
     };
 
     const actual = readAzureDevOpsConfigs(new ConfigReader(config));
 
-    expect(actual).toHaveLength(3);
+    expect(actual).toHaveLength(4);
     expect(actual[0]).toEqual({
       ...provider1,
       path: '/catalog-info.yaml',
@@ -63,5 +74,16 @@ describe('readAzureDevOpsConfigs', () => {
       path: '/catalog-info.yaml',
       id: 'provider3',
     });
+    expect(actual[3]).toEqual({
+      ...provider4,
+      host: 'dev.azure.com',
+      path: '/catalog-info.yaml',
+      repository: '*',
+      id: 'provider4',
+      schedule: {
+        ...provider4.schedule,
+        frequency: Duration.fromISO(provider4.schedule.frequency),
+      },
+    });
   });
 });

plugins/catalog-backend-module-azure/src/providers/config.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks';
 import { Config } from '@backstage/config';
 import { AzureDevOpsConfig } from './types';
 
@@ -42,6 +43,10 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig {
   const repository = config.getOptionalString('repository') || '*';
   const path = config.getOptionalString('path') || '/catalog-info.yaml';
 
+  const schedule = config.has('schedule')
+    ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
+    : undefined;
+
   return {
     id,
     host,
@@ -49,5 +54,6 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig {
     project,
     repository,
     path,
+    schedule,
   };
 }

plugins/catalog-backend-module-azure/src/providers/types.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+
 export type AzureDevOpsConfig = {
   id: string;
   host: string;
@@ -21,4 +23,5 @@ export type AzureDevOpsConfig = {
   project: string;
   repository: string;
   path: string;
+  schedule?: TaskScheduleDefinition;
 };

yarn.lock

@@ -4477,6 +4477,7 @@ __metadata:
     "@backstage/types": "workspace:^"
     "@types/lodash": ^4.14.151
     lodash: ^4.17.21
+    luxon: ^3.0.0
     msw: ^0.47.0
     node-fetch: ^2.6.7
     uuid: ^8.0.0