diff --git a/.changeset/gorgeous-onions-thank.md b/.changeset/gorgeous-onions-thank.md new file mode 100644 index 0000000000..8b41bfa5a0 --- /dev/null +++ b/.changeset/gorgeous-onions-thank.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-aws': patch +--- + +`AwsS3EntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/aws-s3/discovery diff --git a/docs/integrations/aws-s3/discovery.md b/docs/integrations/aws-s3/discovery.md index 9e9e6ebede..748aeb2392 100644 --- a/docs/integrations/aws-s3/discovery.md +++ b/docs/integrations/aws-s3/discovery.md @@ -32,6 +32,11 @@ catalog: bucketName: sample-bucket prefix: prefix/ # optional region: us-east-2 # optional, uses the default region otherwise + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` For simple setups, you can omit the provider ID at the config @@ -47,6 +52,11 @@ catalog: bucketName: sample-bucket prefix: prefix/ # optional region: us-east-2 # optional, uses the default region otherwise + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` As this provider is not one of the default providers, you will first need to install @@ -69,10 +79,13 @@ const builder = await CatalogBuilder.create(env); builder.addEntityProvider( AwsS3EntityProvider.fromConfig(env.config, { logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml schedule: env.scheduler.createScheduledTaskRunner({ frequency: { minutes: 30 }, timeout: { minutes: 3 }, }), + // optional: alternatively, use schedule + scheduler: env.scheduler, }), ); ``` diff --git a/plugins/catalog-backend-module-aws/api-report.md b/plugins/catalog-backend-module-aws/api-report.md index b1f6358979..c5e56818c2 100644 --- a/plugins/catalog-backend-module-aws/api-report.md +++ b/plugins/catalog-backend-module-aws/api-report.md @@ -12,6 +12,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; import { UrlReader } from '@backstage/backend-common'; @@ -77,7 +78,8 @@ export class AwsS3EntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): AwsS3EntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-aws/config.d.ts b/plugins/catalog-backend-module-aws/config.d.ts index 6f84ca6daf..d6efe10c90 100644 --- a/plugins/catalog-backend-module-aws/config.d.ts +++ b/plugins/catalog-backend-module-aws/config.d.ts @@ -14,11 +14,10 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { catalog?: { - /** - * List of processor-specific options and attributes - */ processors?: { /** * AwsOrganizationCloudAccountProcessor configuration @@ -32,9 +31,6 @@ export interface Config { }; }; }; - /** - * List of provider-specific options and attributes - */ providers?: { /** * AwsS3EntityProvider configuration @@ -45,22 +41,23 @@ export interface Config { | { /** * (Required) AWS S3 Bucket Name - * @visibility backend */ bucketName: string; /** * (Optional) AWS S3 Object key prefix * If not set, all keys will be accepted, no filtering will be applied. - * @visibility backend */ prefix?: string; /** * (Optional) AWS Region. * If not set, AWS_REGION environment variable or aws config file will be used. * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html - * @visibility backend */ region?: string; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } | Record< string, @@ -83,6 +80,10 @@ export interface Config { * @visibility backend */ region?: string; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-aws/package.json b/plugins/catalog-backend-module-aws/package.json index 8be3ca3031..e348d76c93 100644 --- a/plugins/catalog-backend-module-aws/package.json +++ b/plugins/catalog-backend-module-aws/package.json @@ -50,6 +50,7 @@ "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", "aws-sdk-mock": "^5.2.1", + "luxon": "^3.0.0", "yaml": "^2.0.0" }, "files": [ diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts index 3d59d738cf..134a3c6338 100644 --- a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts +++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts @@ -15,7 +15,11 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { AwsS3EntityProvider } from './AwsS3EntityProvider'; @@ -83,6 +87,7 @@ describe('AwsS3EntityProvider', () => { expectedBaseUrl: string, names: Record, integrationConfig?: object, + scheduleInConfig?: boolean, ) => { const config = new ConfigReader({ integrations: { @@ -97,15 +102,25 @@ describe('AwsS3EntityProvider', () => { }, }); + const schedulingConfig: Record = {}; + const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), refresh: jest.fn(), }; + if (scheduleInConfig) { + schedulingConfig.scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + } else { + schedulingConfig.schedule = schedule; + } + const provider = AwsS3EntityProvider.fromConfig(config, { + ...schedulingConfig, logger, - schedule, })[0]; expect(provider.getProviderName()).toEqual(`awsS3-provider:${providerId}`); @@ -224,4 +239,81 @@ describe('AwsS3EntityProvider', () => { }, ); }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + awsS3: { + test: { + bucketName: 'bucket-1', + prefix: 'sub/dir/', + region: 'eu-west-1', + }, + }, + }, + }, + }); + + expect(() => + AwsS3EntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + awsS3: { + test: { + bucketName: 'bucket-1', + prefix: 'sub/dir/', + region: 'eu-west-1', + }, + }, + }, + }, + }); + + expect(() => + AwsS3EntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for awsS3-provider:test', + ); + }); + + // eslint-disable-next-line jest/expect-expect + it('single simple provider config with schedule in config', async () => { + return expectMutation( + 'regionalStatic', + { + bucketName: 'bucket-1', + prefix: 'sub/dir/', + region: 'eu-west-1', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + 'https://s3.eu-west-1.amazonaws.com/bucket-1/sub/dir/', + { + 'key1.yaml': 'generated-7f6d5861b0b3401a38b5fe62e6c7ca11da5fd6d8', + 'key2.yaml': 'generated-a290be145586042af7d80715626399c9d661718d', + 'key3.yaml': 'generated-8d75f78ed9fa618ce433b226dc24eeab441f3a2d', + 'key 4.yaml': 'generated-1e0249dcb5805fc2ce6ac2d3c4d2a3ef4f1270c0', + }, + undefined, + true, + ); + }); }); diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts index cee5eea1a4..80bc4d459b 100644 --- a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts +++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; import { @@ -49,7 +49,8 @@ export class AwsS3EntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): AwsS3EntityProvider[] { const providerConfigs = readAwsS3Configs(configRoot); @@ -67,22 +68,35 @@ export class AwsS3EntityProvider implements EntityProvider { throw new Error('No integration found for awsS3'); } - return providerConfigs.map( - providerConfig => - new AwsS3EntityProvider( - providerConfig, - integration, - options.logger, - options.schedule, - ), - ); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + + return providerConfigs.map(providerConfig => { + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for awsS3-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + + return new AwsS3EntityProvider( + providerConfig, + integration, + options.logger, + taskRunner, + ); + }); } private constructor( private readonly config: AwsS3Config, integration: AwsS3Integration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, ) { this.logger = logger.child({ target: this.getProviderName(), @@ -99,13 +113,13 @@ export class AwsS3EntityProvider implements EntityProvider { s3ForcePathStyle: integration.config.s3ForcePathStyle, }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-aws/src/providers/config.test.ts b/plugins/catalog-backend-module-aws/src/providers/config.test.ts index 534d4da35e..fb2f3c0790 100644 --- a/plugins/catalog-backend-module-aws/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-aws/src/providers/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readAwsS3Configs } from './config'; describe('readAwsS3Configs', () => { @@ -54,17 +55,26 @@ describe('readAwsS3Configs', () => { const provider3 = { bucketName: 'bucket-3', }; + const provider4 = { + bucketName: 'bucket-4', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }; const config = { catalog: { providers: { - awsS3: { provider1, provider2, provider3 }, + awsS3: { provider1, provider2, provider3, provider4 }, }, }, }; const actual = readAwsS3Configs(new ConfigReader(config)); - expect(actual).toHaveLength(3); + expect(actual).toHaveLength(4); expect(actual[0]).toEqual({ ...provider1, id: 'provider1', @@ -77,6 +87,14 @@ describe('readAwsS3Configs', () => { ...provider3, id: 'provider3', }); + expect(actual[3]).toEqual({ + ...provider4, + id: 'provider4', + schedule: { + ...provider4.schedule, + frequency: Duration.fromISO(provider4.schedule.frequency), + }, + }); }); it('fails if bucketName is missing', () => { diff --git a/plugins/catalog-backend-module-aws/src/providers/config.ts b/plugins/catalog-backend-module-aws/src/providers/config.ts index 5fdf2351c3..5ef2729bf0 100644 --- a/plugins/catalog-backend-module-aws/src/providers/config.ts +++ b/plugins/catalog-backend-module-aws/src/providers/config.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { AwsS3Config } from './types'; @@ -46,10 +47,15 @@ function readAwsS3Config(id: string, config: Config): AwsS3Config { const region = config.getOptionalString('region'); const prefix = config.getOptionalString('prefix'); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, bucketName, region, prefix, + schedule, }; } diff --git a/plugins/catalog-backend-module-aws/src/providers/types.ts b/plugins/catalog-backend-module-aws/src/providers/types.ts index 3508bf919c..204fa154c6 100644 --- a/plugins/catalog-backend-module-aws/src/providers/types.ts +++ b/plugins/catalog-backend-module-aws/src/providers/types.ts @@ -14,9 +14,12 @@ * limitations under the License. */ +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + export type AwsS3Config = { id: string; bucketName: string; prefix?: string; region?: string; + schedule?: TaskScheduleDefinition; }; diff --git a/yarn.lock b/yarn.lock index ac27eb8215..58cf42441d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4450,6 +4450,7 @@ __metadata: aws-sdk: ^2.840.0 aws-sdk-mock: ^5.2.1 lodash: ^4.17.21 + luxon: ^3.0.0 p-limit: ^3.0.2 uuid: ^8.0.0 winston: ^3.2.1