From d4fea86ea386ce6e8032a0c7a48cf68b1ee19e8e Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Tue, 27 Sep 2022 02:58:22 +0200 Subject: [PATCH 1/3] feat(backend): Add function to read schedule from config New function `readTaskScheduleDefinition` to read `TaskScheduleDefinition` (aka. schedule) from the `Config`. Signed-off-by: Patrick Jungermann --- .changeset/fuzzy-dolls-give.md | 5 + packages/backend-tasks/api-report.md | 18 +++ packages/backend-tasks/src/tasks/index.ts | 2 + ...adTaskScheduleDefinitionFromConfig.test.ts | 119 ++++++++++++++++++ .../readTaskScheduleDefinitionFromConfig.ts | 102 +++++++++++++++ packages/backend-tasks/src/tasks/types.ts | 103 ++++++++++++++- 6 files changed, 346 insertions(+), 3 deletions(-) create mode 100644 .changeset/fuzzy-dolls-give.md create mode 100644 packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.test.ts create mode 100644 packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.ts diff --git a/.changeset/fuzzy-dolls-give.md b/.changeset/fuzzy-dolls-give.md new file mode 100644 index 0000000000..4de4d2e128 --- /dev/null +++ b/.changeset/fuzzy-dolls-give.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-tasks': patch +--- + +Added new function `readTaskScheduleDefinitionFromConfig` to read `TaskScheduleDefinition` (aka. schedule) from the `Config`. diff --git a/packages/backend-tasks/api-report.md b/packages/backend-tasks/api-report.md index c0802fa46c..a60f6dba37 100644 --- a/packages/backend-tasks/api-report.md +++ b/packages/backend-tasks/api-report.md @@ -30,6 +30,11 @@ export interface PluginTaskScheduler { triggerTask(id: string): Promise; } +// @public +export function readTaskScheduleDefinitionFromConfig( + config: Config, +): TaskScheduleDefinition; + // @public export type TaskFunction = | ((abortSignal: AbortSignal_2) => void | Promise) @@ -60,6 +65,19 @@ export interface TaskScheduleDefinition { timeout: Duration | HumanDuration; } +// @public +export interface TaskScheduleDefinitionConfig { + frequency: + | { + cron: string; + } + | string + | HumanDuration; + initialDelay?: string | HumanDuration; + scope?: 'global' | 'local'; + timeout: string | HumanDuration; +} + // @public export class TaskScheduler { constructor(databaseManager: DatabaseManager, logger: Logger); diff --git a/packages/backend-tasks/src/tasks/index.ts b/packages/backend-tasks/src/tasks/index.ts index 990d6332fc..8f30e5c8dd 100644 --- a/packages/backend-tasks/src/tasks/index.ts +++ b/packages/backend-tasks/src/tasks/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +export { readTaskScheduleDefinitionFromConfig } from './readTaskScheduleDefinitionFromConfig'; export { TaskScheduler } from './TaskScheduler'; export type { PluginTaskScheduler, @@ -21,5 +22,6 @@ export type { TaskInvocationDefinition, TaskRunner, TaskScheduleDefinition, + TaskScheduleDefinitionConfig, HumanDuration, } from './types'; diff --git a/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.test.ts b/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.test.ts new file mode 100644 index 0000000000..115ef89168 --- /dev/null +++ b/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.test.ts @@ -0,0 +1,119 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; +import { readTaskScheduleDefinitionFromConfig } from './readTaskScheduleDefinitionFromConfig'; +import { HumanDuration } from './types'; + +describe('readTaskScheduleDefinitionFromConfig', () => { + it('all valid values', () => { + const config = new ConfigReader({ + frequency: { + cron: '0 30 * * * *', + }, + timeout: 'PT3M', + initialDelay: { + minutes: 20, + }, + scope: 'global', + }); + + const result = readTaskScheduleDefinitionFromConfig(config); + + expect((result.frequency as { cron: string }).cron).toBe('0 30 * * * *'); + expect(result.timeout).toEqual(Duration.fromISO('PT3M')); + expect((result.initialDelay as HumanDuration).minutes).toEqual(20); + expect(result.scope).toBe('global'); + }); + + it('all valid required values', () => { + const config = new ConfigReader({ + frequency: { + cron: '0 30 * * * *', + }, + timeout: 'PT3M', + }); + + const result = readTaskScheduleDefinitionFromConfig(config); + + expect((result.frequency as { cron: string }).cron).toBe('0 30 * * * *'); + expect(result.timeout).toEqual(Duration.fromISO('PT3M')); + expect(result.initialDelay).toBeUndefined(); + expect(result.scope).toBeUndefined(); + }); + + it('fail without required frequency', () => { + const config = new ConfigReader({ + timeout: 'PT3M', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + "Missing required config value at 'frequency'", + ); + }); + + it('fail without required timeout', () => { + const config = new ConfigReader({ + frequency: 'PT30M', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + "Missing required config value at 'timeout'", + ); + }); + + it('invalid frequency value', () => { + const config = new ConfigReader({ + frequency: { + invalid: 'value', + }, + timeout: 'PT3M', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + 'HumanDuration needs at least one of', + ); + }); + + it('frequency value with additional invalid prop', () => { + const config = new ConfigReader({ + frequency: { + minutes: 20, + invalid: 'value', + }, + timeout: 'PT3M', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + 'HumanDuration does not contain properties: invalid', + ); + }); + + it('invalid scope value', () => { + const config = new ConfigReader({ + frequency: { + years: 2, + }, + timeout: 'PT3M', + scope: 'invalid', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + 'Only "global" or "local" are allowed for TaskScheduleDefinition.scope, but got: invalid', + ); + }); +}); diff --git a/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.ts b/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.ts new file mode 100644 index 0000000000..3267d8ef14 --- /dev/null +++ b/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { JsonObject } from '@backstage/types'; +import { HumanDuration, TaskScheduleDefinition } from './types'; +import { Duration } from 'luxon'; + +const propsOfHumanDuration = [ + 'years', + 'months', + 'weeks', + 'days', + 'hours', + 'minutes', + 'seconds', + 'milliseconds', +]; + +function convertToHumanDuration(config: Config, key: string): HumanDuration { + const props = config.getConfig(key).keys(); + if (!props.find(prop => propsOfHumanDuration.includes(prop))) { + throw new Error( + `HumanDuration needs at least one of: ${propsOfHumanDuration}`, + ); + } + + const invalidProps = props.filter( + prop => !propsOfHumanDuration.includes(prop), + ); + if (invalidProps.length > 0) { + throw new Error( + `HumanDuration does not contain properties: ${invalidProps}`, + ); + } + + return config.get(key) as HumanDuration; +} + +function readDuration(config: Config, key: string): Duration | HumanDuration { + return typeof config.get(key) === 'string' + ? Duration.fromISO(config.getString(key)) + : convertToHumanDuration(config, key); +} + +function readCronOrDuration( + config: Config, + key: string, +): { cron: string } | Duration | HumanDuration { + const value = config.get(key); + if (typeof value === 'object' && (value as { cron?: string }).cron) { + return value as { cron: string }; + } + + return readDuration(config, key); +} + +/** + * Reads a TaskScheduleDefinition from a Config. + * Expects the config not to be the root config, + * but the config for the definition. + * + * @param config - config for a TaskScheduleDefinition. + * @public + */ +export function readTaskScheduleDefinitionFromConfig( + config: Config, +): TaskScheduleDefinition { + const frequency = readCronOrDuration(config, 'frequency'); + const timeout = readDuration(config, 'timeout'); + + const initialDelay = config.has('initialDelay') + ? readDuration(config, 'initialDelay') + : undefined; + + const scope = config.getOptionalString('scope'); + if (scope && !['global', 'local'].includes(scope)) { + throw new Error( + `Only "global" or "local" are allowed for TaskScheduleDefinition.scope, but got: ${scope}`, + ); + } + + return { + frequency, + timeout, + initialDelay, + scope: scope as 'global' | 'local' | undefined, + }; +} diff --git a/packages/backend-tasks/src/tasks/types.ts b/packages/backend-tasks/src/tasks/types.ts index 86c0c5c462..92ec045605 100644 --- a/packages/backend-tasks/src/tasks/types.ts +++ b/packages/backend-tasks/src/tasks/types.ts @@ -58,7 +58,7 @@ export interface TaskScheduleDefinition { * * @remarks * - * This is a best effort value; under some circumstances there can be + * This is the best effort value; under some circumstances there can be * deviations. For example, if the task runtime is longer than the frequency * and the timeout has not been given or not been exceeded yet, the next * invocation of this task will be delayed until after the previous one @@ -112,7 +112,7 @@ export interface TaskScheduleDefinition { * collaborating on a task that has its `scope` field set to `'global'`, then * you may still see the task being processed by other long-lived workers, * while any given single worker is in its initial sleep delay time e.g. after - * a deployment. Therefore this parameter is not useful for "globally" pausing + * a deployment. Therefore, this parameter is not useful for "globally" pausing * work; its main intended use is for individual machines to get a chance to * reach some equilibrium at startup before triggering heavy batch workloads. */ @@ -128,7 +128,104 @@ export interface TaskScheduleDefinition { * attempt to ensure that only one worker machine runs the task at a time, * according to the given cadence. This means that as the number of worker * hosts increases, the invocation frequency of this task will not go up. - * Instead the load is spread randomly across hosts. This setting is useful + * Instead, the load is spread randomly across hosts. This setting is useful + * for tasks that access shared resources, for example catalog ingestion tasks + * where you do not want many machines to repeatedly import the same data and + * trample over each other. + * + * When the scope is set to `'local'`, there is no concurrency control across + * hosts. Each host runs the task according to the given cadence similarly to + * `setInterval`, but the runtime ensures that there are no overlapping runs. + * + * @defaultValue 'global' + */ + scope?: 'global' | 'local'; +} + +/** + * Config options for {@link TaskScheduleDefinition} + * that control the scheduling of a task. + * + * @public + */ +export interface TaskScheduleDefinitionConfig { + /** + * How often you want the task to run. The system does its best to avoid + * overlapping invocations. + * + * @remarks + * + * This is the best effort value; under some circumstances there can be + * deviations. For example, if the task runtime is longer than the frequency + * and the timeout has not been given or not been exceeded yet, the next + * invocation of this task will be delayed until after the previous one + * finishes. + * + * This is a required field. + */ + frequency: + | { + /** + * A crontab style string. + * + * @remarks + * + * Overview: + * + * ``` + * ┌────────────── second (optional) + * │ ┌──────────── minute + * │ │ ┌────────── hour + * │ │ │ ┌──────── day of month + * │ │ │ │ ┌────── month + * │ │ │ │ │ ┌──── day of week + * │ │ │ │ │ │ + * │ │ │ │ │ │ + * * * * * * * + * ``` + */ + cron: string; + } + | string + | HumanDuration; + + /** + * The maximum amount of time that a single task invocation can take, before + * it's considered timed out and gets "released" such that a new invocation + * is permitted to take place (possibly, then, on a different worker). + */ + timeout: string | HumanDuration; + + /** + * The amount of time that should pass before the first invocation happens. + * + * @remarks + * + * This can be useful in cold start scenarios to stagger or delay some heavy + * compute jobs. If no value is given for this field then the first invocation + * will happen as soon as possible according to the cadence. + * + * NOTE: This is a per-worker delay. If you have a cluster of workers all + * collaborating on a task that has its `scope` field set to `'global'`, then + * you may still see the task being processed by other long-lived workers, + * while any given single worker is in its initial sleep delay time e.g. after + * a deployment. Therefore, this parameter is not useful for "globally" pausing + * work; its main intended use is for individual machines to get a chance to + * reach some equilibrium at startup before triggering heavy batch workloads. + */ + initialDelay?: string | HumanDuration; + + /** + * Sets the scope of concurrency control / locking to apply for invocations of + * this task. + * + * @remarks + * + * When the scope is set to the default value `'global'`, the scheduler will + * attempt to ensure that only one worker machine runs the task at a time, + * according to the given cadence. This means that as the number of worker + * hosts increases, the invocation frequency of this task will not go up. + * Instead, the load is spread randomly across hosts. This setting is useful * for tasks that access shared resources, for example catalog ingestion tasks * where you do not want many machines to repeatedly import the same data and * trample over each other. From f66e696e7bd3640e4daad7d7a2c32b66e7713b41 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Tue, 27 Sep 2022 02:58:22 +0200 Subject: [PATCH 2/3] feat(catalog/bitbucketCloud): Add option to configure schedule via `app-config.yaml` Signed-off-by: Patrick Jungermann --- .changeset/brown-grapes-battle.md | 8 ++ docs/integrations/bitbucketCloud/discovery.md | 25 +++++-- .../api-report.md | 4 +- .../config.d.ts | 10 +++ .../package.json | 1 + .../src/BitbucketCloudEntityProvider.test.ts | 75 ++++++++++++++++++- .../src/BitbucketCloudEntityProvider.ts | 40 ++++++---- ...BitbucketCloudEntityProviderConfig.test.ts | 27 ++++++- .../src/BitbucketCloudEntityProviderConfig.ts | 10 +++ yarn.lock | 1 + 10 files changed, 180 insertions(+), 21 deletions(-) create mode 100644 .changeset/brown-grapes-battle.md diff --git a/.changeset/brown-grapes-battle.md b/.changeset/brown-grapes-battle.md new file mode 100644 index 0000000000..f94c938b02 --- /dev/null +++ b/.changeset/brown-grapes-battle.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-cloud': minor +--- + +Bitbucket Cloud provider: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/bitbucketCloud/discovery diff --git a/docs/integrations/bitbucketCloud/discovery.md b/docs/integrations/bitbucketCloud/discovery.md index 6609707868..c5a17bb2c9 100644 --- a/docs/integrations/bitbucketCloud/discovery.md +++ b/docs/integrations/bitbucketCloud/discovery.md @@ -37,6 +37,7 @@ And then add the entity provider to your catalog builder: + builder.addEntityProvider( + BitbucketCloudEntityProvider.fromConfig(env.config, { + logger: env.logger, ++ // optional: alternatively, configure via app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, @@ -67,24 +68,38 @@ catalog: filters: # optional projectKey: '^apis-.*$' # optional; RegExp repoSlug: '^service-.*$' # optional; RegExp + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } workspace: workspace-name ``` > **Note:** It is possible but certainly not recommended to skip the provider ID level. > If you do so, `default` will be used as provider ID. -- **catalogPath** _(optional)_: +- **`catalogPath`** _(optional)_: Default: `/catalog-info.yaml`. Path where to look for `catalog-info.yaml` files. When started with `/`, it is an absolute path from the repo root. It supports values as allowed by the `path` filter/modifier [at Bitbucket Cloud's code search](https://confluence.atlassian.com/bitbucket/code-search-in-bitbucket-873876782.html#Search-Pathmodifier). -- **filters** _(optional)_: - - **projectKey** _(optional)_: +- **`filters`** _(optional)_: + - **`projectKey`** _(optional)_: Regular expression used to filter results based on the project key. - - **repoSlug** _(optional)_: + - **`repoSlug`** _(optional)_: Regular expression used to filter results based on the repo slug. -- **workspace**: +- **`schedule`** _(optional)_: + - **`frequency`**: + How often you want the task to run. The system does its best to avoid overlapping invocations. + - **`timeout`**: + The maximum amount of time that a single task invocation can take. + - **`initialDelay`** _(optional)_: + The amount of time that should pass before the first invocation happens. + - **`scope`** _(optional)_: + `'global'` or `'local'`. Sets the scope of concurrency control. +- **`workspace`**: Name of your organization account/workspace. If you want to add multiple workspaces, you need to add one provider config each. diff --git a/plugins/catalog-backend-module-bitbucket-cloud/api-report.md b/plugins/catalog-backend-module-bitbucket-cloud/api-report.md index b886cc224f..2f8b4186d1 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/api-report.md +++ b/plugins/catalog-backend-module-bitbucket-cloud/api-report.md @@ -7,6 +7,7 @@ import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; // @public @@ -18,7 +19,8 @@ export class BitbucketCloudEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): BitbucketCloudEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts index 29289bb70e..8fa58413c0 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { catalog?: { /** @@ -53,6 +55,10 @@ export interface Config { */ projectKey?: RegExp; }; + /** + * (Optional) TaskScheduleDefinition for the discovery. + */ + schedule?: TaskScheduleDefinitionConfig; } | Record< string, @@ -83,6 +89,10 @@ export interface Config { */ projectKey?: RegExp; }; + /** + * (Optional) TaskScheduleDefinition for the discovery. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-bitbucket-cloud/package.json b/plugins/catalog-backend-module-bitbucket-cloud/package.json index d36d7c653e..6079f4fe14 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/package.json +++ b/plugins/catalog-backend-module-bitbucket-cloud/package.json @@ -44,6 +44,7 @@ "@backstage/backend-common": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", + "luxon": "^3.0.0", "msw": "^0.47.0" }, "files": [ diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts index 641150acbf..7d780cd064 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts @@ -15,7 +15,11 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; @@ -77,6 +81,75 @@ describe('BitbucketCloudEntityProvider', () => { ); }); + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + workspace: 'test-ws', + }, + }, + }, + }); + + expect(() => + BitbucketCloudEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided.'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = jest.fn() as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + workspace: 'test-ws', + }, + }, + }, + }); + + expect(() => + BitbucketCloudEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for bitbucketCloud-provider:default.', + ); + }); + + it('single simple provider config with schedule in config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + workspace: 'test-ws', + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }); + + const providers = BitbucketCloudEntityProvider.fromConfig(config, { + logger, + scheduler, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'bitbucketCloud-provider:default', + ); + }); + it('multiple provider configs', () => { const schedule = new PersistingTaskRunner(); const config = new ConfigReader({ diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts index 74d642ed9b..f3334a6574 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { BitbucketCloudIntegration, @@ -58,7 +58,8 @@ export class BitbucketCloudEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): BitbucketCloudEntityProvider[] { const integrations = ScmIntegrations.fromConfig(config); @@ -69,29 +70,42 @@ export class BitbucketCloudEntityProvider implements EntityProvider { throw new Error('No integration for bitbucket.org available'); } - return readProviderConfigs(config).map( - providerConfig => - new BitbucketCloudEntityProvider( - providerConfig, - integration, - options.logger, - options.schedule, - ), - ); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + + return readProviderConfigs(config).map(providerConfig => { + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for bitbucketCloud-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + + return new BitbucketCloudEntityProvider( + providerConfig, + integration, + options.logger, + taskRunner, + ); + }); } private constructor( config: BitbucketCloudEntityProviderConfig, integration: BitbucketCloudIntegration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, ) { this.client = BitbucketCloudClient.fromConfig(integration.config); this.config = config; this.logger = logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); } private createScheduleFn(schedule: TaskRunner): () => Promise { diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.test.ts index a42bd2cde7..5a2cc99437 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readProviderConfigs } from './BitbucketCloudEntityProviderConfig'; describe('readProviderConfigs', () => { @@ -68,13 +69,22 @@ describe('readProviderConfigs', () => { repoSlug: 'repoSlug.*filter', }, }, + providerWithSchedule: { + workspace: 'test-ws5', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(4); + expect(providerConfigs).toHaveLength(5); expect(providerConfigs[0]).toEqual({ id: 'providerWorkspaceOnly', workspace: 'test-ws1', @@ -111,5 +121,20 @@ describe('readProviderConfigs', () => { repoSlug: /^repoSlug.*filter$/, }, }); + expect(providerConfigs[4]).toEqual({ + id: 'providerWithSchedule', + workspace: 'test-ws5', + catalogPath: '/catalog-info.yaml', + filters: { + projectKey: undefined, + repoSlug: undefined, + }, + schedule: { + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 3, + }, + }, + }); }); }); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.ts index 95e995f8c3..def88b0951 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.ts @@ -14,6 +14,10 @@ * limitations under the License. */ +import { + readTaskScheduleDefinitionFromConfig, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; @@ -27,6 +31,7 @@ export type BitbucketCloudEntityProviderConfig = { projectKey?: RegExp; repoSlug?: RegExp; }; + schedule?: TaskScheduleDefinition; }; export function readProviderConfigs( @@ -61,6 +66,10 @@ function readProviderConfig( const projectKeyPattern = config.getOptionalString('filters.projectKey'); const repoSlugPattern = config.getOptionalString('filters.repoSlug'); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, catalogPath, @@ -71,6 +80,7 @@ function readProviderConfig( : undefined, repoSlug: repoSlugPattern ? compileRegExp(repoSlugPattern) : undefined, }, + schedule, }; } diff --git a/yarn.lock b/yarn.lock index 1178b3d940..ad1fe8b20a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4462,6 +4462,7 @@ __metadata: "@backstage/integration": "workspace:^" "@backstage/plugin-bitbucket-cloud-common": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + luxon: ^3.0.0 msw: ^0.47.0 uuid: ^8.0.0 winston: ^3.2.1 From a9b91d39bb61b35cdbc0598892673c1a4e3fe27a Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Tue, 27 Sep 2022 03:16:21 +0200 Subject: [PATCH 3/3] feat(catalog/bitbucketCloud): Add backend plugin Add `bitbucketCloudEntityProviderCatalogModule` (new backend-plugin-api, alpha). Signed-off-by: Patrick Jungermann --- .changeset/giant-pianos-repeat.md | 5 ++ .../api-report.md | 6 ++ .../package.json | 14 ++-- .../src/index.ts | 1 + .../BitbucketCloudCatalogModule.test.ts | 84 +++++++++++++++++++ ...tbucketCloudEntityProviderCatalogModule.ts | 52 ++++++++++++ yarn.lock | 2 + 7 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 .changeset/giant-pianos-repeat.md create mode 100644 plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudCatalogModule.test.ts create mode 100644 plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.ts diff --git a/.changeset/giant-pianos-repeat.md b/.changeset/giant-pianos-repeat.md new file mode 100644 index 0000000000..d83ce285e3 --- /dev/null +++ b/.changeset/giant-pianos-repeat.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch +--- + +Add `bitbucketCloudCatalogModule` (new backend-plugin-api, alpha). diff --git a/plugins/catalog-backend-module-bitbucket-cloud/api-report.md b/plugins/catalog-backend-module-bitbucket-cloud/api-report.md index 2f8b4186d1..162ff8f67e 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/api-report.md +++ b/plugins/catalog-backend-module-bitbucket-cloud/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; @@ -30,4 +31,9 @@ export class BitbucketCloudEntityProvider implements EntityProvider { // (undocumented) refresh(logger: Logger): Promise; } + +// @alpha (undocumented) +export const bitbucketCloudEntityProviderCatalogModule: ( + options?: undefined, +) => BackendFeature; ``` diff --git a/plugins/catalog-backend-module-bitbucket-cloud/package.json b/plugins/catalog-backend-module-bitbucket-cloud/package.json index 6079f4fe14..6b340ab0f1 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/package.json +++ b/plugins/catalog-backend-module-bitbucket-cloud/package.json @@ -7,6 +7,7 @@ "license": "Apache-2.0", "publishConfig": { "access": "public", + "alphaTypes": "dist/index.alpha.d.ts", "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, @@ -23,20 +24,22 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", + "start": "backstage-cli package start", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", - "clean": "backstage-cli package clean", - "start": "backstage-cli package start" + "clean": "backstage-cli package clean" }, "dependencies": { + "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/config": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-bitbucket-cloud-common": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", "uuid": "^8.0.0", "winston": "^3.2.1" }, @@ -48,8 +51,9 @@ "msw": "^0.47.0" }, "files": [ - "dist", - "config.d.ts" + "alpha", + "config.d.ts", + "dist" ], "configSchema": "config.d.ts" } diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/index.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/index.ts index 1c15ad4e8f..c1f04b8877 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/index.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/index.ts @@ -21,3 +21,4 @@ */ export { BitbucketCloudEntityProvider } from './BitbucketCloudEntityProvider'; +export { bitbucketCloudEntityProviderCatalogModule } from './service/BitbucketCloudEntityProviderCatalogModule'; diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudCatalogModule.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudCatalogModule.test.ts new file mode 100644 index 0000000000..6307bee282 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudCatalogModule.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { + PluginTaskScheduler, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { startTestBackend } from '@backstage/backend-test-utils'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { bitbucketCloudEntityProviderCatalogModule } from './BitbucketCloudEntityProviderCatalogModule'; +import { Duration } from 'luxon'; +import { BitbucketCloudEntityProvider } from '../BitbucketCloudEntityProvider'; + +describe('bitbucketCloudEntityProviderCatalogModule', () => { + it('should register provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = { + createScheduledTaskRunner: (schedule: TaskScheduleDefinition) => { + usedSchedule = schedule; + return runner; + }, + } as unknown as PluginTaskScheduler; + + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + workspace: 'test-ws', + }, + }, + }, + }); + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + services: [ + [configServiceRef, config], + [loggerServiceRef, getVoidLogger()], + [schedulerServiceRef, scheduler], + ], + features: [bitbucketCloudEntityProviderCatalogModule()], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'bitbucketCloud-provider:default', + ); + expect(runner).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.ts new file mode 100644 index 0000000000..d137bd7166 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + configServiceRef, + createBackendModule, + loggerServiceRef, + loggerToWinstonLogger, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { BitbucketCloudEntityProvider } from '../BitbucketCloudEntityProvider'; + +/** + * @alpha + */ +export const bitbucketCloudEntityProviderCatalogModule = createBackendModule({ + pluginId: 'catalog', + moduleId: 'bitbucketCloudEntityProvider', + register(env) { + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + config: configServiceRef, + logger: loggerServiceRef, + scheduler: schedulerServiceRef, + }, + async init({ catalog, config, logger, scheduler }) { + const winstonLogger = loggerToWinstonLogger(logger); + const providers = BitbucketCloudEntityProvider.fromConfig(config, { + logger: winstonLogger, + scheduler, + }); + + catalog.addEntityProvider(providers); + }, + }); + }, +}); diff --git a/yarn.lock b/yarn.lock index ad1fe8b20a..4aa9986616 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4455,6 +4455,7 @@ __metadata: resolution: "@backstage/plugin-catalog-backend-module-bitbucket-cloud@workspace:plugins/catalog-backend-module-bitbucket-cloud" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" @@ -4462,6 +4463,7 @@ __metadata: "@backstage/integration": "workspace:^" "@backstage/plugin-bitbucket-cloud-common": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" luxon: ^3.0.0 msw: ^0.47.0 uuid: ^8.0.0