diff --git a/.changeset/ten-pens-draw.md b/.changeset/ten-pens-draw.md new file mode 100644 index 0000000000..be35a6f3b0 --- /dev/null +++ b/.changeset/ten-pens-draw.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-gerrit': patch +--- + +`GerritEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/gerrit/discovery diff --git a/docs/integrations/gerrit/discovery.md b/docs/integrations/gerrit/discovery.md index 04fa3dac2a..7ed05d2b3b 100644 --- a/docs/integrations/gerrit/discovery.md +++ b/docs/integrations/gerrit/discovery.md @@ -32,10 +32,13 @@ const builder = await CatalogBuilder.create(env); builder.addEntityProvider( GerritEntityProvider.fromConfig(env.config, { logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml schedule: env.scheduler.createScheduledTaskRunner({ frequency: { minutes: 30 }, timeout: { minutes: 3 }, }), + // optional: alternatively, use schedule + scheduler: env.scheduler, }), ); ``` @@ -54,6 +57,11 @@ catalog: host: gerrit-your-company.com branch: master # Optional query: 'state=ACTIVE&prefix=webapps' + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } backend: host: gerrit-your-company.com branch: master # Optional @@ -62,8 +70,8 @@ catalog: The provider configuration is composed of three parts: -- host, the host of the Gerrit integration to use. -- branch, the branch where we will look for catalog entities (defaults to "master"). -- query, this string is directly used as the argument to the "List Project" API. - Typically you will want to have some filter here to exclude projects that will +- **`host`**: the host of the Gerrit integration to use. +- **`branch`** _(optional)_: the branch where we will look for catalog entities (defaults to "master"). +- **`query`**: this string is directly used as the argument to the "List Project" API. + Typically, you will want to have some filter here to exclude projects that will never contain any catalog files. diff --git a/plugins/catalog-backend-module-gerrit/api-report.md b/plugins/catalog-backend-module-gerrit/api-report.md index 7eae94c2a8..1dee786da1 100644 --- a/plugins/catalog-backend-module-gerrit/api-report.md +++ b/plugins/catalog-backend-module-gerrit/api-report.md @@ -7,6 +7,7 @@ import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; // @public (undocumented) @@ -18,7 +19,8 @@ export class GerritEntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): GerritEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-gerrit/package.json b/plugins/catalog-backend-module-gerrit/package.json index e3665a96ae..1cebc25bc8 100644 --- a/plugins/catalog-backend-module-gerrit/package.json +++ b/plugins/catalog-backend-module-gerrit/package.json @@ -44,7 +44,8 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", - "@types/fs-extra": "^9.0.1" + "@types/fs-extra": "^9.0.1", + "luxon": "^3.0.0" }, "files": [ "dist", diff --git a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts index d6ad0ff668..5d258ee94f 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts @@ -15,14 +15,18 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { ConfigReader } from '@backstage/config'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; -import { rest } from 'msw'; import fs from 'fs-extra'; -import path from 'path'; +import { rest } from 'msw'; import { setupServer } from 'msw/node'; +import path from 'path'; import { GerritEntityProvider } from './GerritEntityProvider'; const server = setupServer(); @@ -207,4 +211,93 @@ describe('GerritEntityProvider', () => { }), ).toThrow(/No gerrit integration/); }); + + it('fail without schedule and scheduler', () => { + expect(() => + GerritEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + expect(() => + GerritEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for gerrit-provider:active-training', + ); + }); + + it('discovers projects from the api with schedule in config', async () => { + const configWithSchedule = new ConfigReader({ + catalog: { + providers: { + gerrit: { + 'active-training': { + host: 'g.com', + query: 'state=ACTIVE&prefix=training', + branch: 'main', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + }, + }, + }, + integrations: { + gerrit: [ + { + host: 'g.com', + baseUrl: 'https://g.com/gerrit', + gitilesBaseUrl: 'https:/g.com/gitiles', + }, + ], + }, + }); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + + const repoBuffer = fs.readFileSync( + path.resolve(__dirname, '__fixtures__/listProjectsBody.txt'), + ); + const expected = getJsonFixture('expectedProviderEntities.json'); + + server.use( + rest.get('https://g.com/gerrit/projects/', (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.body(repoBuffer), + ), + ), + ); + + const provider = GerritEntityProvider.fromConfig(configWithSchedule, { + logger, + scheduler, + })[0]; + expect(provider.getProviderName()).toEqual( + 'gerrit-provider:active-training', + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('gerrit-provider:active-training:refresh'); + await (taskDef.fn as () => Promise)(); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith( + expected, + ); + }); }); diff --git a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.ts b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.ts index beb4052073..ec08dbe4be 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { InputError } from '@backstage/errors'; import { @@ -49,9 +49,14 @@ export class GerritEntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): GerritEntityProvider[] { + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + const providerConfigs = readGerritConfigs(configRoot); const integrations = ScmIntegrations.fromConfig(configRoot).gerrit; const providers: GerritEntityProvider[] = []; @@ -63,12 +68,23 @@ export class GerritEntityProvider implements EntityProvider { `No gerrit integration found that matches host ${providerConfig.host}`, ); } + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for gerrit-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + providers.push( new GerritEntityProvider( providerConfig, integration, options.logger, - options.schedule, + taskRunner, ), ); }); @@ -79,14 +95,14 @@ export class GerritEntityProvider implements EntityProvider { config: GerritProviderConfig, integration: GerritIntegration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, ) { this.config = config; this.integration = integration; this.logger = logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); } getProviderName(): string { @@ -98,10 +114,10 @@ export class GerritEntityProvider implements EntityProvider { await this.scheduleFn(); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-gerrit/src/providers/config.test.ts b/plugins/catalog-backend-module-gerrit/src/providers/config.test.ts index 0422b4e4ea..ecf68d9f3c 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readGerritConfigs } from './config'; describe('readGerritConfigs', () => { @@ -29,12 +30,24 @@ describe('readGerritConfigs', () => { query: 'state=ACTIVE', branch: 'main', }; + const provider3 = { + host: 'gerrit1.com', + query: 'state=ACTIVE', + branch: 'main', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }; const config = { catalog: { providers: { gerrit: { 'active-g1': provider1, 'active-g2': provider2, + 'active-g3': provider3, }, }, }, @@ -42,10 +55,19 @@ describe('readGerritConfigs', () => { const actual = readGerritConfigs(new ConfigReader(config)); - expect(actual).toHaveLength(2); + expect(actual).toHaveLength(3); expect(actual[0]).toEqual({ ...provider1, id: 'active-g1' }); expect(actual[1]).toEqual({ ...provider2, id: 'active-g2' }); + expect(actual[2]).toEqual({ + ...provider3, + id: 'active-g3', + schedule: { + ...provider3.schedule, + frequency: Duration.fromISO(provider3.schedule.frequency), + }, + }); }); + it('provides default values', () => { const provider = { host: 'gerrit1.com', diff --git a/plugins/catalog-backend-module-gerrit/src/providers/config.ts b/plugins/catalog-backend-module-gerrit/src/providers/config.ts index 7a28ddd8ea..dd7ee9ec43 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/config.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/config.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GerritProviderConfig } from './types'; @@ -22,11 +23,16 @@ function readGerritConfig(id: string, config: Config): GerritProviderConfig { const host = config.getString('host'); const query = config.getString('query'); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { branch, host, id, query, + schedule, }; } diff --git a/plugins/catalog-backend-module-gerrit/src/providers/types.ts b/plugins/catalog-backend-module-gerrit/src/providers/types.ts index 1e96bd1ee4..4a8329df86 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/types.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + export type GerritProjectInfo = { id: string; name: string; @@ -28,4 +30,5 @@ export type GerritProviderConfig = { query: string; id: string; branch?: string; + schedule?: TaskScheduleDefinition; }; diff --git a/yarn.lock b/yarn.lock index 1551240610..bfb1485395 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4569,6 +4569,7 @@ __metadata: "@backstage/plugin-catalog-backend": "workspace:^" "@types/fs-extra": ^9.0.1 fs-extra: 10.1.0 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 uuid: ^8.0.0