From 68f7f5a857f1b47edad74dba5ccf231447e65b1b Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 10:11:29 +0200 Subject: [PATCH] feat(catalog/bitbucketServer): Add option to configure schedule via `app-config.yaml` Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/tasty-scissors-tickle.md | 8 + .../integrations/bitbucketServer/discovery.md | 36 ++-- .../api-report.md | 4 +- .../config.d.ts | 13 +- .../package.json | 1 + .../BitbucketServerEntityProvider.test.ts | 171 +++++++++++++++++- .../BitbucketServerEntityProvider.ts | 52 ++++-- ...itbucketServerEntityProviderConfig.test.ts | 27 ++- .../BitbucketServerEntityProviderConfig.ts | 10 + yarn.lock | 1 + 10 files changed, 285 insertions(+), 38 deletions(-) create mode 100644 .changeset/tasty-scissors-tickle.md diff --git a/.changeset/tasty-scissors-tickle.md b/.changeset/tasty-scissors-tickle.md new file mode 100644 index 0000000000..745172df36 --- /dev/null +++ b/.changeset/tasty-scissors-tickle.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-server': patch +--- + +`BitbucketServerEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/bitbucketServer/discovery diff --git a/docs/integrations/bitbucketServer/discovery.md b/docs/integrations/bitbucketServer/discovery.md index 57e52d1dae..c4288b33ec 100644 --- a/docs/integrations/bitbucketServer/discovery.md +++ b/docs/integrations/bitbucketServer/discovery.md @@ -37,10 +37,13 @@ And then add the entity provider to your catalog builder: + builder.addEntityProvider( + BitbucketServerEntityProvider.fromConfig(env.config, { + logger: env.logger, ++ // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), ++ // optional: alternatively, use schedule ++ scheduler: env.scheduler, + }), + ); @@ -66,19 +69,33 @@ catalog: filters: # optional projectKey: '^apis-.*$' # optional; RegExp repoSlug: '^service-.*$' # optional; RegExp + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` -- **host**: +- **`host`**: The host of the Bitbucket Server instance, **note**: the host needs to registered as an integration as well, see [location](locations.md). - **`catalogPath`** _(optional)_: Default: `/catalog-info.yaml`. Path where to look for `catalog-info.yaml` files. When started with `/`, it is an absolute path from the repo root. -- **filters** _(optional)_: +- **`filters`** _(optional)_: - **`projectKey`** _(optional)_: Regular expression used to filter results based on the project key. - - **repoSlug** _(optional)_: + - **`repoSlug`** _(optional)_: Regular expression used to filter results based on the repo slug. +- **`schedule`** _(optional)_: + - **`frequency`**: + How often you want the task to run. The system does its best to avoid overlapping invocations. + - **`timeout`**: + The maximum amount of time that a single task invocation can take. + - **`initialDelay`** _(optional)_: + The amount of time that should pass before the first invocation happens. + - **`scope`** _(optional)_: + `'global'` or `'local'`. Sets the scope of concurrency control. ## Custom location processing @@ -93,18 +110,15 @@ repository. ```typescript const provider = BitbucketServerEntityProvider.fromConfig(env.config, { logger: env.logger, - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 30 }, - timeout: { minutes: 3 }, - }), + schedule: env.scheduler, parser: async function* customLocationParser(options: { - location: LocationSpec, - client: BitbucketServerClient, + location: LocationSpec; + client: BitbucketServerClient; }) { // Custom logic for interpreting the matching repository // See defaultBitbucketServerLocationParser for an example - } -); + }, +}); ``` ## Alternative diff --git a/plugins/catalog-backend-module-bitbucket-server/api-report.md b/plugins/catalog-backend-module-bitbucket-server/api-report.md index 7d95e148a3..6c840883af 100644 --- a/plugins/catalog-backend-module-bitbucket-server/api-report.md +++ b/plugins/catalog-backend-module-bitbucket-server/api-report.md @@ -10,6 +10,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Response as Response_2 } from 'node-fetch'; import { TaskRunner } from '@backstage/backend-tasks'; @@ -55,8 +56,9 @@ export class BitbucketServerEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; parser?: BitbucketServerLocationParser; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): BitbucketServerEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-bitbucket-server/config.d.ts b/plugins/catalog-backend-module-bitbucket-server/config.d.ts index d8a09c92a4..67f694d419 100644 --- a/plugins/catalog-backend-module-bitbucket-server/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-server/config.d.ts @@ -14,11 +14,10 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { catalog?: { - /** - * List of provider-specific options and attributes - */ providers?: { /** * BitbucketServerEntityProvider configuration @@ -48,6 +47,10 @@ export interface Config { */ projectKey?: RegExp; }; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } | Record< string, @@ -73,6 +76,10 @@ export interface Config { */ projectKey?: RegExp; }; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-bitbucket-server/package.json b/plugins/catalog-backend-module-bitbucket-server/package.json index 2490b79ffa..021b531cea 100644 --- a/plugins/catalog-backend-module-bitbucket-server/package.json +++ b/plugins/catalog-backend-module-bitbucket-server/package.json @@ -46,6 +46,7 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", + "luxon": "^3.0.0", "msw": "^0.47.0" }, "files": [ diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts index 39689b6c1d..20299e3452 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts @@ -15,14 +15,18 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; -import { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import { BitbucketServerPagedResponse } from '../lib/BitbucketServerClient'; +import { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider'; +import { BitbucketServerPagedResponse } from '../lib'; class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -367,4 +371,163 @@ describe('BitbucketServerEntityProvider', () => { entities: expectedEntities, }); }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + host: 'bitbucket.mycompany.com', + }, + }, + }, + integrations: { + bitbucketServer: [ + { + host: 'bitbucket.mycompany.com', + }, + ], + }, + }); + + expect(() => + BitbucketServerEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + host: 'bitbucket.mycompany.com', + }, + }, + }, + integrations: { + bitbucketServer: [ + { + host: 'bitbucket.mycompany.com', + }, + ], + }, + }); + + expect(() => + BitbucketServerEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for bitbucketServer-provider:default', + ); + }); + + it('apply full update with schedule in config', async () => { + const host = 'bitbucket.mycompany.com'; + const config = new ConfigReader({ + integrations: { + bitbucketServer: [ + { + host: host, + }, + ], + }, + catalog: { + providers: { + bitbucketServer: { + mainProvider: { + host: host, + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = BitbucketServerEntityProvider.fromConfig(config, { + logger, + scheduler, + })[0]; + expect(provider.getProviderName()).toEqual( + 'bitbucketServer-provider:mainProvider', + ); + + setupStubs( + [ + { key: 'project-test', repos: ['repo-test'] }, + { key: 'other-project', repos: ['other-repo'] }, + ], + `https://${host}`, + ); + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('bitbucketServer-provider:mainProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`, + 'backstage.io/managed-by-origin-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`, + }, + name: 'generated-77f4323822420990f8c3e3c981d38c2dec4ae3a6', + }, + spec: { + presence: 'optional', + target: `https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`, + type: 'url', + }, + }, + locationKey: 'bitbucketServer-provider:mainProvider', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + 'backstage.io/managed-by-origin-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + }, + name: 'generated-d8d4944c30c2906dfee172ddda9537f9893b2c0f', + }, + spec: { + presence: 'optional', + target: `https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + type: 'url', + }, + }, + locationKey: 'bitbucketServer-provider:mainProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); }); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts index 92cecf642a..447b71ee58 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts @@ -14,29 +14,29 @@ * limitations under the License. */ +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; +import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { InputError } from '@backstage/errors'; +import { + BitbucketServerIntegration, + ScmIntegrations, +} from '@backstage/integration'; import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; -import { Config } from '@backstage/config'; -import { TaskRunner } from '@backstage/backend-tasks'; import * as uuid from 'uuid'; -import { - BitbucketServerLocationParser, - defaultBitbucketServerLocationParser, -} from './BitbucketServerLocationParser'; -import { - BitbucketServerIntegration, - ScmIntegrations, -} from '@backstage/integration'; import { BitbucketServerClient, paginated } from '../lib'; -import { InputError } from '@backstage/errors'; import { BitbucketServerEntityProviderConfig, readProviderConfigs, } from './BitbucketServerEntityProviderConfig'; -import { Entity } from '@backstage/catalog-model'; +import { + BitbucketServerLocationParser, + defaultBitbucketServerLocationParser, +} from './BitbucketServerLocationParser'; /** * Discovers catalog files located in Bitbucket Server. @@ -58,12 +58,17 @@ export class BitbucketServerEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; parser?: BitbucketServerLocationParser; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): BitbucketServerEntityProvider[] { const integrations = ScmIntegrations.fromConfig(config); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + return readProviderConfigs(config).map(providerConfig => { const integration = integrations.bitbucketServer.byHost( providerConfig.host, @@ -73,11 +78,22 @@ export class BitbucketServerEntityProvider implements EntityProvider { `No BitbucketServer integration found that matches host ${providerConfig.host}`, ); } + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for bitbucketServer-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + return new BitbucketServerEntityProvider( providerConfig, integration, options.logger, - options.schedule, + taskRunner, options.parser, ); }); @@ -87,7 +103,7 @@ export class BitbucketServerEntityProvider implements EntityProvider { config: BitbucketServerEntityProviderConfig, integration: BitbucketServerIntegration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, parser?: BitbucketServerLocationParser, ) { this.integration = integration; @@ -96,13 +112,13 @@ export class BitbucketServerEntityProvider implements EntityProvider { this.logger = logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts index 712b26cbb0..af8639566f 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readProviderConfigs } from './BitbucketServerEntityProviderConfig'; describe('readProviderConfigs', () => { @@ -63,13 +64,22 @@ describe('readProviderConfigs', () => { host: 'bitbucket2.mycompany.com', catalogPath: 'custom/path/catalog-info.yaml', }, + thirdProvider: { + host: 'bitbucket3.mycompany.com', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(2); + expect(providerConfigs).toHaveLength(3); expect(providerConfigs[0]).toEqual({ id: 'mainProvider', catalogPath: '/catalog-info.yaml', @@ -88,6 +98,21 @@ describe('readProviderConfigs', () => { repoSlug: undefined, }, }); + expect(providerConfigs[2]).toEqual({ + id: 'thirdProvider', + catalogPath: '/catalog-info.yaml', + host: 'bitbucket3.mycompany.com', + filters: { + projectKey: undefined, + repoSlug: undefined, + }, + schedule: { + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 3, + }, + }, + }); }); it('single provider config with filters', () => { diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts index 8a3decab43..b2ed7a8556 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts @@ -14,6 +14,10 @@ * limitations under the License. */ +import { + readTaskScheduleDefinitionFromConfig, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; @@ -27,6 +31,7 @@ export type BitbucketServerEntityProviderConfig = { projectKey?: RegExp; repoSlug?: RegExp; }; + schedule?: TaskScheduleDefinition; }; export function readProviderConfigs( @@ -60,6 +65,10 @@ function readProviderConfig( const projectKeyPattern = config.getOptionalString('filters.projectKey'); const repoSlugPattern = config.getOptionalString('filters.repoSlug'); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, host, @@ -68,5 +77,6 @@ function readProviderConfig( projectKey: projectKeyPattern ? new RegExp(projectKeyPattern) : undefined, repoSlug: repoSlugPattern ? new RegExp(repoSlugPattern) : undefined, }, + schedule, }; } diff --git a/yarn.lock b/yarn.lock index 074a6fc323..b90c1a515d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4522,6 +4522,7 @@ __metadata: "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" "@types/node-fetch": ^2.5.12 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 uuid: ^8.0.0