From 81cedb503351d0df1449f7ffd5ee8c41973147ad Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 12:13:18 +0200 Subject: [PATCH] feat(catalog/gitlab): Add option to configure schedule via `app-config.yaml` Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/flat-kangaroos-kiss.md | 8 ++ docs/integrations/gitlab/discovery.md | 8 ++ .../api-report.md | 4 +- .../catalog-backend-module-gitlab/config.d.ts | 13 +- .../package.json | 3 +- .../src/lib/types.ts | 3 + .../GitlabDiscoveryEntityProvider.test.ts | 117 +++++++++++++++++- .../GitlabDiscoveryEntityProvider.ts | 38 ++++-- .../src/providers/config.test.ts | 45 +++++++ .../src/providers/config.ts | 6 + yarn.lock | 1 + 11 files changed, 225 insertions(+), 21 deletions(-) create mode 100644 .changeset/flat-kangaroos-kiss.md diff --git a/.changeset/flat-kangaroos-kiss.md b/.changeset/flat-kangaroos-kiss.md new file mode 100644 index 0000000000..a79c8621b1 --- /dev/null +++ b/.changeset/flat-kangaroos-kiss.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +`GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/gitlab/discovery diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index df5aadcab4..dc0b911b3d 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -25,6 +25,11 @@ catalog: group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole project will be scanned entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` projectPattern: /[\s\S]*/ # Optional. Filters found projects based on provided patter. Defaults to `/[\s\S]*/`, what means to not filter anything + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` As this provider is not one of the default providers, you will first need to install @@ -47,10 +52,13 @@ const builder = await CatalogBuilder.create(env); builder.addEntityProvider( ...GitlabDiscoveryEntityProvider.fromConfig(env.config, { logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml schedule: env.scheduler.createScheduledTaskRunner({ frequency: { minutes: 30 }, timeout: { minutes: 3 }, }), + // optional: alternatively, use schedule + scheduler: env.scheduler, }), ); ``` diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index 1009da8209..bd7b13efc3 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -10,6 +10,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; // @public @@ -21,7 +22,8 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): GitlabDiscoveryEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-gitlab/config.d.ts b/plugins/catalog-backend-module-gitlab/config.d.ts index 82f1a5db7c..ac75a04c2a 100644 --- a/plugins/catalog-backend-module-gitlab/config.d.ts +++ b/plugins/catalog-backend-module-gitlab/config.d.ts @@ -14,11 +14,10 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { catalog?: { - /** - * List of provider-specific options and attributes - */ providers?: { /** * GitlabDiscoveryEntityProvider configuration @@ -28,27 +27,27 @@ export interface Config { { /** * (Required) Gitlab's host name. - * @visibility backend */ host: string; /** * (Optional) Gitlab's group[/subgroup] where the discovery is done. * If not defined the whole project will be scanned. - * @visibility backend */ group?: string; /** * (Optional) Default branch to read the catalog-info.yaml file. * If not set, 'master' will be used. - * @visibility backend */ branch?: string; /** * (Optional) The name used for the catalog file. * If not set, 'catalog-info.yaml' will be used. - * @visibility backend */ entityFilename?: string; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 90ce82b01e..e537fa5e04 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -50,7 +50,8 @@ "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", - "@types/uuid": "^8.0.0" + "@types/uuid": "^8.0.0", + "luxon": "^3.0.0" }, "files": [ "config.d.ts", diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 8a3c33c8f2..86fe1aa385 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + export type GitlabGroupDescription = { id: number; web_url: string; @@ -36,4 +38,5 @@ export type GitlabProviderConfig = { branch: string; catalogFile: string; projectPattern: RegExp; + schedule?: TaskScheduleDefinition; }; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index 3f3c4e7f91..ae19edc48c 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -15,13 +15,17 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; -import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; +import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -338,4 +342,111 @@ describe('GitlabDiscoveryEntityProvider', () => { ], }); }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + }, + }, + }, + }); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + }, + }, + }, + }); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id', + ); + }); + + it('single simple provider config with schedule in config', async () => { + const schedule = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }); + const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index ffb123edf7..cb4629b820 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -14,14 +14,16 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; import { EntityProvider, EntityProviderConnection, + LocationSpec, + locationSpecToLocationEntity, } from '@backstage/plugin-catalog-backend'; -import { LocationSpec } from '@backstage/plugin-catalog-backend'; +import * as uuid from 'uuid'; import { Logger } from 'winston'; import { GitLabClient, @@ -30,8 +32,6 @@ import { paginated, readGitlabConfigs, } from '../lib'; -import * as uuid from 'uuid'; -import { locationSpecToLocationEntity } from '@backstage/plugin-catalog-backend'; type Result = { scanned: number; @@ -51,8 +51,16 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { static fromConfig( config: Config, - options: { logger: Logger; schedule: TaskRunner }, + options: { + logger: Logger; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; + }, ): GitlabDiscoveryEntityProvider[] { + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + const providerConfigs = readGitlabConfigs(config); const integrations = ScmIntegrations.fromConfig(config).gitlab; const providers: GitlabDiscoveryEntityProvider[] = []; @@ -64,11 +72,23 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { `No gitlab integration found that matches host ${providerConfig.host}`, ); } + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + providers.push( new GitlabDiscoveryEntityProvider({ ...options, config: providerConfig, integration, + taskRunner, }), ); }); @@ -79,14 +99,14 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { config: GitlabProviderConfig; integration: GitLabIntegration; logger: Logger; - schedule: TaskRunner; + taskRunner: TaskRunner; }) { this.config = options.config; this.integration = options.integration; this.logger = options.logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(options.schedule); + this.scheduleFn = this.createScheduleFn(options.taskRunner); } getProviderName(): string { @@ -98,10 +118,10 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { await this.scheduleFn(); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index cb36e5166c..6b337bf7cd 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readGitlabConfigs } from './config'; describe('config', () => { @@ -53,6 +54,7 @@ describe('config', () => { host: 'host', catalogFile: 'catalog-info.yaml', projectPattern: /[\s\S]*/, + schedule: undefined, }), ); }); @@ -83,6 +85,49 @@ describe('config', () => { host: 'host', catalogFile: 'custom-file.yaml', projectPattern: /[\s\S]*/, + schedule: undefined, + }), + ); + }); + + it('valid config with schedule', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + group: 'group', + host: 'host', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + }, + }, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(1); + result.forEach(r => + expect(r).toStrictEqual({ + id: 'test', + group: 'group', + branch: 'master', + host: 'host', + catalogFile: 'catalog-info.yaml', + projectPattern: /[\s\S]*/, + schedule: { + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 3, + }, + initialDelay: undefined, + scope: undefined, + }, }), ); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index 392922860f..3dfc00745e 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GitlabProviderConfig } from '../lib'; @@ -35,6 +36,10 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { config.getOptionalString('projectPattern') ?? /[\s\S]*/, ); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, group, @@ -42,6 +47,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { host, catalogFile, projectPattern, + schedule, }; } diff --git a/yarn.lock b/yarn.lock index 787cb2a94a..4c1f1a9bb6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4622,6 +4622,7 @@ __metadata: "@types/lodash": ^4.14.151 "@types/uuid": ^8.0.0 lodash: ^4.17.21 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 uuid: ^8.0.0