feat(catalog/awsS3): Add option to configure schedule via app-config.yaml

Relates-to: PR #13859
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>

.changeset/gorgeous-onions-thank.md

@@ -0,0 +1,8 @@
+---
+'@backstage/plugin-catalog-backend-module-aws': patch
+---
+
+`AwsS3EntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
+
+Please find how to configure the schedule at the config at
+https://backstage.io/docs/integrations/aws-s3/discovery

docs/integrations/aws-s3/discovery.md

@@ -32,6 +32,11 @@ catalog:
         bucketName: sample-bucket
         prefix: prefix/ # optional
         region: us-east-2 # optional, uses the default region otherwise
+        schedule: # optional; same options as in TaskScheduleDefinition
+          # supports cron, ISO duration, "human duration" as used in code
+          frequency: { minutes: 30 }
+          # supports ISO duration, "human duration" as used in code
+          timeout: { minutes: 3 }
 ```
 
 For simple setups, you can omit the provider ID at the config
@@ -47,6 +52,11 @@ catalog:
       bucketName: sample-bucket
       prefix: prefix/ # optional
       region: us-east-2 # optional, uses the default region otherwise
+      schedule: # optional; same options as in TaskScheduleDefinition
+        # supports cron, ISO duration, "human duration" as used in code
+        frequency: { minutes: 30 }
+        # supports ISO duration, "human duration" as used in code
+        timeout: { minutes: 3 }
 ```
 
 As this provider is not one of the default providers, you will first need to install
@@ -69,10 +79,13 @@ const builder = await CatalogBuilder.create(env);
 builder.addEntityProvider(
   AwsS3EntityProvider.fromConfig(env.config, {
     logger: env.logger,
+    // optional: alternatively, use scheduler with schedule defined in app-config.yaml
     schedule: env.scheduler.createScheduledTaskRunner({
       frequency: { minutes: 30 },
       timeout: { minutes: 3 },
     }),
+    // optional: alternatively, use schedule
+    scheduler: env.scheduler,
   }),
 );
 ```

plugins/catalog-backend-module-aws/api-report.md

@@ -12,6 +12,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
+import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { TaskRunner } from '@backstage/backend-tasks';
 import { UrlReader } from '@backstage/backend-common';
 
@@ -77,7 +78,8 @@ export class AwsS3EntityProvider implements EntityProvider {
     configRoot: Config,
     options: {
       logger: Logger;
-      schedule: TaskRunner;
+      schedule?: TaskRunner;
+      scheduler?: PluginTaskScheduler;
     },
   ): AwsS3EntityProvider[];
   // (undocumented)

plugins/catalog-backend-module-aws/config.d.ts

@@ -14,11 +14,10 @@
  * limitations under the License.
  */
 
+import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks';
+
 export interface Config {
   catalog?: {
-    /**
-     * List of processor-specific options and attributes
-     */
     processors?: {
       /**
        * AwsOrganizationCloudAccountProcessor configuration
@@ -32,9 +31,6 @@ export interface Config {
         };
       };
     };
-    /**
-     * List of provider-specific options and attributes
-     */
     providers?: {
       /**
        * AwsS3EntityProvider configuration
@@ -45,22 +41,23 @@ export interface Config {
         | {
             /**
              * (Required) AWS S3 Bucket Name
-             * @visibility backend
              */
             bucketName: string;
             /**
              * (Optional) AWS S3 Object key prefix
              * If not set, all keys will be accepted, no filtering will be applied.
-             * @visibility backend
              */
             prefix?: string;
             /**
              * (Optional) AWS Region.
              * If not set, AWS_REGION environment variable or aws config file will be used.
              * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
-             * @visibility backend
              */
             region?: string;
+            /**
+             * (Optional) TaskScheduleDefinition for the refresh.
+             */
+            schedule?: TaskScheduleDefinitionConfig;
           }
         | Record<
             string,
@@ -83,6 +80,10 @@ export interface Config {
                * @visibility backend
                */
               region?: string;
+              /**
+               * (Optional) TaskScheduleDefinition for the refresh.
+               */
+              schedule?: TaskScheduleDefinitionConfig;
             }
           >;
     };

plugins/catalog-backend-module-aws/package.json

@@ -50,6 +50,7 @@
     "@backstage/cli": "workspace:^",
     "@types/lodash": "^4.14.151",
     "aws-sdk-mock": "^5.2.1",
+    "luxon": "^3.0.0",
     "yaml": "^2.0.0"
   },
   "files": [

plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts

@@ -15,7 +15,11 @@
  */
 
 import { getVoidLogger } from '@backstage/backend-common';
-import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
+import {
+  PluginTaskScheduler,
+  TaskInvocationDefinition,
+  TaskRunner,
+} from '@backstage/backend-tasks';
 import { ConfigReader } from '@backstage/config';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { AwsS3EntityProvider } from './AwsS3EntityProvider';
@@ -83,6 +87,7 @@ describe('AwsS3EntityProvider', () => {
     expectedBaseUrl: string,
     names: Record<string, string>,
     integrationConfig?: object,
+    scheduleInConfig?: boolean,
   ) => {
     const config = new ConfigReader({
       integrations: {
@@ -97,15 +102,25 @@ describe('AwsS3EntityProvider', () => {
       },
     });
 
+    const schedulingConfig: Record<string, any> = {};
+
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
       refresh: jest.fn(),
     };
 
+    if (scheduleInConfig) {
+      schedulingConfig.scheduler = {
+        createScheduledTaskRunner: (_: any) => schedule,
+      } as unknown as PluginTaskScheduler;
+    } else {
+      schedulingConfig.schedule = schedule;
+    }
+
     const provider = AwsS3EntityProvider.fromConfig(config, {
+      ...schedulingConfig,
       logger,
-      schedule,
     })[0];
     expect(provider.getProviderName()).toEqual(`awsS3-provider:${providerId}`);
 
@@ -224,4 +239,81 @@ describe('AwsS3EntityProvider', () => {
       },
     );
   });
+
+  it('fail without schedule and scheduler', () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          awsS3: {
+            test: {
+              bucketName: 'bucket-1',
+              prefix: 'sub/dir/',
+              region: 'eu-west-1',
+            },
+          },
+        },
+      },
+    });
+
+    expect(() =>
+      AwsS3EntityProvider.fromConfig(config, {
+        logger,
+      }),
+    ).toThrow('Either schedule or scheduler must be provided');
+  });
+
+  it('fail with scheduler but no schedule config', () => {
+    const scheduler = {
+      createScheduledTaskRunner: (_: any) => jest.fn(),
+    } as unknown as PluginTaskScheduler;
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          awsS3: {
+            test: {
+              bucketName: 'bucket-1',
+              prefix: 'sub/dir/',
+              region: 'eu-west-1',
+            },
+          },
+        },
+      },
+    });
+
+    expect(() =>
+      AwsS3EntityProvider.fromConfig(config, {
+        logger,
+        scheduler,
+      }),
+    ).toThrow(
+      'No schedule provided neither via code nor config for awsS3-provider:test',
+    );
+  });
+
+  // eslint-disable-next-line jest/expect-expect
+  it('single simple provider config with schedule in config', async () => {
+    return expectMutation(
+      'regionalStatic',
+      {
+        bucketName: 'bucket-1',
+        prefix: 'sub/dir/',
+        region: 'eu-west-1',
+        schedule: {
+          frequency: 'PT30M',
+          timeout: {
+            minutes: 3,
+          },
+        },
+      },
+      'https://s3.eu-west-1.amazonaws.com/bucket-1/sub/dir/',
+      {
+        'key1.yaml': 'generated-7f6d5861b0b3401a38b5fe62e6c7ca11da5fd6d8',
+        'key2.yaml': 'generated-a290be145586042af7d80715626399c9d661718d',
+        'key3.yaml': 'generated-8d75f78ed9fa618ce433b226dc24eeab441f3a2d',
+        'key 4.yaml': 'generated-1e0249dcb5805fc2ce6ac2d3c4d2a3ef4f1270c0',
+      },
+      undefined,
+      true,
+    );
+  });
 });

plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { TaskRunner } from '@backstage/backend-tasks';
+import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
 import { Config } from '@backstage/config';
 import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
 import {
@@ -49,7 +49,8 @@ export class AwsS3EntityProvider implements EntityProvider {
     configRoot: Config,
     options: {
       logger: Logger;
-      schedule: TaskRunner;
+      schedule?: TaskRunner;
+      scheduler?: PluginTaskScheduler;
     },
   ): AwsS3EntityProvider[] {
     const providerConfigs = readAwsS3Configs(configRoot);
@@ -67,22 +68,35 @@ export class AwsS3EntityProvider implements EntityProvider {
       throw new Error('No integration found for awsS3');
     }
 
-    return providerConfigs.map(
-      providerConfig =>
-        new AwsS3EntityProvider(
-          providerConfig,
-          integration,
-          options.logger,
-          options.schedule,
-        ),
-    );
+    if (!options.schedule && !options.scheduler) {
+      throw new Error('Either schedule or scheduler must be provided.');
+    }
+
+    return providerConfigs.map(providerConfig => {
+      if (!options.schedule && !providerConfig.schedule) {
+        throw new Error(
+          `No schedule provided neither via code nor config for awsS3-provider:${providerConfig.id}.`,
+        );
+      }
+
+      const taskRunner =
+        options.schedule ??
+        options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
+
+      return new AwsS3EntityProvider(
+        providerConfig,
+        integration,
+        options.logger,
+        taskRunner,
+      );
+    });
   }
 
   private constructor(
     private readonly config: AwsS3Config,
     integration: AwsS3Integration,
     logger: Logger,
-    schedule: TaskRunner,
+    taskRunner: TaskRunner,
   ) {
     this.logger = logger.child({
       target: this.getProviderName(),
@@ -99,13 +113,13 @@ export class AwsS3EntityProvider implements EntityProvider {
       s3ForcePathStyle: integration.config.s3ForcePathStyle,
     });
 
-    this.scheduleFn = this.createScheduleFn(schedule);
+    this.scheduleFn = this.createScheduleFn(taskRunner);
   }
 
-  private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
+  private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
     return async () => {
       const taskId = `${this.getProviderName()}:refresh`;
-      return schedule.run({
+      return taskRunner.run({
         id: taskId,
         fn: async () => {
           const logger = this.logger.child({

plugins/catalog-backend-module-aws/src/providers/config.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { ConfigReader } from '@backstage/config';
+import { Duration } from 'luxon';
 import { readAwsS3Configs } from './config';
 
 describe('readAwsS3Configs', () => {
@@ -54,17 +55,26 @@ describe('readAwsS3Configs', () => {
     const provider3 = {
       bucketName: 'bucket-3',
     };
+    const provider4 = {
+      bucketName: 'bucket-4',
+      schedule: {
+        frequency: 'PT30M',
+        timeout: {
+          minutes: 3,
+        },
+      },
+    };
     const config = {
       catalog: {
         providers: {
-          awsS3: { provider1, provider2, provider3 },
+          awsS3: { provider1, provider2, provider3, provider4 },
         },
       },
     };
 
     const actual = readAwsS3Configs(new ConfigReader(config));
 
-    expect(actual).toHaveLength(3);
+    expect(actual).toHaveLength(4);
     expect(actual[0]).toEqual({
       ...provider1,
       id: 'provider1',
@@ -77,6 +87,14 @@ describe('readAwsS3Configs', () => {
       ...provider3,
       id: 'provider3',
     });
+    expect(actual[3]).toEqual({
+      ...provider4,
+      id: 'provider4',
+      schedule: {
+        ...provider4.schedule,
+        frequency: Duration.fromISO(provider4.schedule.frequency),
+      },
+    });
   });
 
   it('fails if bucketName is missing', () => {

plugins/catalog-backend-module-aws/src/providers/config.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks';
 import { Config } from '@backstage/config';
 import { AwsS3Config } from './types';
 
@@ -46,10 +47,15 @@ function readAwsS3Config(id: string, config: Config): AwsS3Config {
   const region = config.getOptionalString('region');
   const prefix = config.getOptionalString('prefix');
 
+  const schedule = config.has('schedule')
+    ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
+    : undefined;
+
   return {
     id,
     bucketName,
     region,
     prefix,
+    schedule,
   };
 }

plugins/catalog-backend-module-aws/src/providers/types.ts

@@ -14,9 +14,12 @@
  * limitations under the License.
  */
 
+import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+
 export type AwsS3Config = {
   id: string;
   bucketName: string;
   prefix?: string;
   region?: string;
+  schedule?: TaskScheduleDefinition;
 };

yarn.lock

@@ -4450,6 +4450,7 @@ __metadata:
     aws-sdk: ^2.840.0
     aws-sdk-mock: ^5.2.1
     lodash: ^4.17.21
+    luxon: ^3.0.0
     p-limit: ^3.0.2
     uuid: ^8.0.0
     winston: ^3.2.1