From bae3617be5a84aaca38bf84c06dd4338946237f3 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Thu, 6 Oct 2022 23:51:18 +0200 Subject: [PATCH 01/11] feat(catalog/awsS3): Add option to configure schedule via `app-config.yaml` Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/gorgeous-onions-thank.md | 8 ++ docs/integrations/aws-s3/discovery.md | 13 +++ .../catalog-backend-module-aws/api-report.md | 4 +- .../catalog-backend-module-aws/config.d.ts | 19 ++-- .../catalog-backend-module-aws/package.json | 1 + .../src/providers/AwsS3EntityProvider.test.ts | 96 ++++++++++++++++++- .../src/providers/AwsS3EntityProvider.ts | 44 ++++++--- .../src/providers/config.test.ts | 22 ++++- .../src/providers/config.ts | 6 ++ .../src/providers/types.ts | 3 + yarn.lock | 1 + 11 files changed, 188 insertions(+), 29 deletions(-) create mode 100644 .changeset/gorgeous-onions-thank.md diff --git a/.changeset/gorgeous-onions-thank.md b/.changeset/gorgeous-onions-thank.md new file mode 100644 index 0000000000..8b41bfa5a0 --- /dev/null +++ b/.changeset/gorgeous-onions-thank.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-aws': patch +--- + +`AwsS3EntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/aws-s3/discovery diff --git a/docs/integrations/aws-s3/discovery.md b/docs/integrations/aws-s3/discovery.md index 9e9e6ebede..748aeb2392 100644 --- a/docs/integrations/aws-s3/discovery.md +++ b/docs/integrations/aws-s3/discovery.md @@ -32,6 +32,11 @@ catalog: bucketName: sample-bucket prefix: prefix/ # optional region: us-east-2 # optional, uses the default region otherwise + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` For simple setups, you can omit the provider ID at the config @@ -47,6 +52,11 @@ catalog: bucketName: sample-bucket prefix: prefix/ # optional region: us-east-2 # optional, uses the default region otherwise + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` As this provider is not one of the default providers, you will first need to install @@ -69,10 +79,13 @@ const builder = await CatalogBuilder.create(env); builder.addEntityProvider( AwsS3EntityProvider.fromConfig(env.config, { logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml schedule: env.scheduler.createScheduledTaskRunner({ frequency: { minutes: 30 }, timeout: { minutes: 3 }, }), + // optional: alternatively, use schedule + scheduler: env.scheduler, }), ); ``` diff --git a/plugins/catalog-backend-module-aws/api-report.md b/plugins/catalog-backend-module-aws/api-report.md index b1f6358979..c5e56818c2 100644 --- a/plugins/catalog-backend-module-aws/api-report.md +++ b/plugins/catalog-backend-module-aws/api-report.md @@ -12,6 +12,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; import { UrlReader } from '@backstage/backend-common'; @@ -77,7 +78,8 @@ export class AwsS3EntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): AwsS3EntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-aws/config.d.ts b/plugins/catalog-backend-module-aws/config.d.ts index 6f84ca6daf..d6efe10c90 100644 --- a/plugins/catalog-backend-module-aws/config.d.ts +++ b/plugins/catalog-backend-module-aws/config.d.ts @@ -14,11 +14,10 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { catalog?: { - /** - * List of processor-specific options and attributes - */ processors?: { /** * AwsOrganizationCloudAccountProcessor configuration @@ -32,9 +31,6 @@ export interface Config { }; }; }; - /** - * List of provider-specific options and attributes - */ providers?: { /** * AwsS3EntityProvider configuration @@ -45,22 +41,23 @@ export interface Config { | { /** * (Required) AWS S3 Bucket Name - * @visibility backend */ bucketName: string; /** * (Optional) AWS S3 Object key prefix * If not set, all keys will be accepted, no filtering will be applied. - * @visibility backend */ prefix?: string; /** * (Optional) AWS Region. * If not set, AWS_REGION environment variable or aws config file will be used. * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html - * @visibility backend */ region?: string; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } | Record< string, @@ -83,6 +80,10 @@ export interface Config { * @visibility backend */ region?: string; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-aws/package.json b/plugins/catalog-backend-module-aws/package.json index 8be3ca3031..e348d76c93 100644 --- a/plugins/catalog-backend-module-aws/package.json +++ b/plugins/catalog-backend-module-aws/package.json @@ -50,6 +50,7 @@ "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", "aws-sdk-mock": "^5.2.1", + "luxon": "^3.0.0", "yaml": "^2.0.0" }, "files": [ diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts index 3d59d738cf..134a3c6338 100644 --- a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts +++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts @@ -15,7 +15,11 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { AwsS3EntityProvider } from './AwsS3EntityProvider'; @@ -83,6 +87,7 @@ describe('AwsS3EntityProvider', () => { expectedBaseUrl: string, names: Record, integrationConfig?: object, + scheduleInConfig?: boolean, ) => { const config = new ConfigReader({ integrations: { @@ -97,15 +102,25 @@ describe('AwsS3EntityProvider', () => { }, }); + const schedulingConfig: Record = {}; + const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), refresh: jest.fn(), }; + if (scheduleInConfig) { + schedulingConfig.scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + } else { + schedulingConfig.schedule = schedule; + } + const provider = AwsS3EntityProvider.fromConfig(config, { + ...schedulingConfig, logger, - schedule, })[0]; expect(provider.getProviderName()).toEqual(`awsS3-provider:${providerId}`); @@ -224,4 +239,81 @@ describe('AwsS3EntityProvider', () => { }, ); }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + awsS3: { + test: { + bucketName: 'bucket-1', + prefix: 'sub/dir/', + region: 'eu-west-1', + }, + }, + }, + }, + }); + + expect(() => + AwsS3EntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + awsS3: { + test: { + bucketName: 'bucket-1', + prefix: 'sub/dir/', + region: 'eu-west-1', + }, + }, + }, + }, + }); + + expect(() => + AwsS3EntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for awsS3-provider:test', + ); + }); + + // eslint-disable-next-line jest/expect-expect + it('single simple provider config with schedule in config', async () => { + return expectMutation( + 'regionalStatic', + { + bucketName: 'bucket-1', + prefix: 'sub/dir/', + region: 'eu-west-1', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + 'https://s3.eu-west-1.amazonaws.com/bucket-1/sub/dir/', + { + 'key1.yaml': 'generated-7f6d5861b0b3401a38b5fe62e6c7ca11da5fd6d8', + 'key2.yaml': 'generated-a290be145586042af7d80715626399c9d661718d', + 'key3.yaml': 'generated-8d75f78ed9fa618ce433b226dc24eeab441f3a2d', + 'key 4.yaml': 'generated-1e0249dcb5805fc2ce6ac2d3c4d2a3ef4f1270c0', + }, + undefined, + true, + ); + }); }); diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts index cee5eea1a4..80bc4d459b 100644 --- a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts +++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; import { @@ -49,7 +49,8 @@ export class AwsS3EntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): AwsS3EntityProvider[] { const providerConfigs = readAwsS3Configs(configRoot); @@ -67,22 +68,35 @@ export class AwsS3EntityProvider implements EntityProvider { throw new Error('No integration found for awsS3'); } - return providerConfigs.map( - providerConfig => - new AwsS3EntityProvider( - providerConfig, - integration, - options.logger, - options.schedule, - ), - ); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + + return providerConfigs.map(providerConfig => { + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for awsS3-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + + return new AwsS3EntityProvider( + providerConfig, + integration, + options.logger, + taskRunner, + ); + }); } private constructor( private readonly config: AwsS3Config, integration: AwsS3Integration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, ) { this.logger = logger.child({ target: this.getProviderName(), @@ -99,13 +113,13 @@ export class AwsS3EntityProvider implements EntityProvider { s3ForcePathStyle: integration.config.s3ForcePathStyle, }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-aws/src/providers/config.test.ts b/plugins/catalog-backend-module-aws/src/providers/config.test.ts index 534d4da35e..fb2f3c0790 100644 --- a/plugins/catalog-backend-module-aws/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-aws/src/providers/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readAwsS3Configs } from './config'; describe('readAwsS3Configs', () => { @@ -54,17 +55,26 @@ describe('readAwsS3Configs', () => { const provider3 = { bucketName: 'bucket-3', }; + const provider4 = { + bucketName: 'bucket-4', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }; const config = { catalog: { providers: { - awsS3: { provider1, provider2, provider3 }, + awsS3: { provider1, provider2, provider3, provider4 }, }, }, }; const actual = readAwsS3Configs(new ConfigReader(config)); - expect(actual).toHaveLength(3); + expect(actual).toHaveLength(4); expect(actual[0]).toEqual({ ...provider1, id: 'provider1', @@ -77,6 +87,14 @@ describe('readAwsS3Configs', () => { ...provider3, id: 'provider3', }); + expect(actual[3]).toEqual({ + ...provider4, + id: 'provider4', + schedule: { + ...provider4.schedule, + frequency: Duration.fromISO(provider4.schedule.frequency), + }, + }); }); it('fails if bucketName is missing', () => { diff --git a/plugins/catalog-backend-module-aws/src/providers/config.ts b/plugins/catalog-backend-module-aws/src/providers/config.ts index 5fdf2351c3..5ef2729bf0 100644 --- a/plugins/catalog-backend-module-aws/src/providers/config.ts +++ b/plugins/catalog-backend-module-aws/src/providers/config.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { AwsS3Config } from './types'; @@ -46,10 +47,15 @@ function readAwsS3Config(id: string, config: Config): AwsS3Config { const region = config.getOptionalString('region'); const prefix = config.getOptionalString('prefix'); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, bucketName, region, prefix, + schedule, }; } diff --git a/plugins/catalog-backend-module-aws/src/providers/types.ts b/plugins/catalog-backend-module-aws/src/providers/types.ts index 3508bf919c..204fa154c6 100644 --- a/plugins/catalog-backend-module-aws/src/providers/types.ts +++ b/plugins/catalog-backend-module-aws/src/providers/types.ts @@ -14,9 +14,12 @@ * limitations under the License. */ +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + export type AwsS3Config = { id: string; bucketName: string; prefix?: string; region?: string; + schedule?: TaskScheduleDefinition; }; diff --git a/yarn.lock b/yarn.lock index ac27eb8215..58cf42441d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4450,6 +4450,7 @@ __metadata: aws-sdk: ^2.840.0 aws-sdk-mock: ^5.2.1 lodash: ^4.17.21 + luxon: ^3.0.0 p-limit: ^3.0.2 uuid: ^8.0.0 winston: ^3.2.1 From defb389ecdc98d6c37725feb5d56bdfbcb3e1391 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 00:06:53 +0200 Subject: [PATCH 02/11] feat(catalog/awsS3): Add backend plugin Add `awsS3EntityProviderCatalogModule` (new backend-plugin-api, alpha). Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/kind-emus-juggle.md | 5 ++ .../catalog-backend-module-aws/api-report.md | 6 ++ .../catalog-backend-module-aws/package.json | 15 ++-- .../catalog-backend-module-aws/src/index.ts | 1 + .../AwsS3EntityProviderCatalogModule.test.ts | 84 +++++++++++++++++++ .../AwsS3EntityProviderCatalogModule.ts | 53 ++++++++++++ yarn.lock | 3 + 7 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 .changeset/kind-emus-juggle.md create mode 100644 plugins/catalog-backend-module-aws/src/service/AwsS3EntityProviderCatalogModule.test.ts create mode 100644 plugins/catalog-backend-module-aws/src/service/AwsS3EntityProviderCatalogModule.ts diff --git a/.changeset/kind-emus-juggle.md b/.changeset/kind-emus-juggle.md new file mode 100644 index 0000000000..2a99deef81 --- /dev/null +++ b/.changeset/kind-emus-juggle.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-aws': patch +--- + +Add `awsS3EntityProviderCatalogModule` (new backend-plugin-api, alpha). diff --git a/plugins/catalog-backend-module-aws/api-report.md b/plugins/catalog-backend-module-aws/api-report.md index c5e56818c2..a911fb5ebb 100644 --- a/plugins/catalog-backend-module-aws/api-report.md +++ b/plugins/catalog-backend-module-aws/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorParser } from '@backstage/plugin-catalog-backend'; @@ -87,4 +88,9 @@ export class AwsS3EntityProvider implements EntityProvider { // (undocumented) refresh(logger: Logger): Promise; } + +// @alpha +export const awsS3EntityProviderCatalogModule: ( + options?: undefined, +) => BackendFeature; ``` diff --git a/plugins/catalog-backend-module-aws/package.json b/plugins/catalog-backend-module-aws/package.json index e348d76c93..56fdf96f14 100644 --- a/plugins/catalog-backend-module-aws/package.json +++ b/plugins/catalog-backend-module-aws/package.json @@ -7,6 +7,7 @@ "license": "Apache-2.0", "publishConfig": { "access": "public", + "alphaTypes": "dist/index.alpha.d.ts", "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, @@ -23,22 +24,24 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", + "start": "backstage-cli package start", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", - "clean": "backstage-cli package clean", - "start": "backstage-cli package start" + "clean": "backstage-cli package clean" }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", "@backstage/types": "workspace:^", "aws-sdk": "^2.840.0", "lodash": "^4.17.21", @@ -47,6 +50,7 @@ "winston": "^3.2.1" }, "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", "aws-sdk-mock": "^5.2.1", @@ -54,8 +58,9 @@ "yaml": "^2.0.0" }, "files": [ - "dist", - "config.d.ts" + "alpha", + "config.d.ts", + "dist" ], "configSchema": "config.d.ts" } diff --git a/plugins/catalog-backend-module-aws/src/index.ts b/plugins/catalog-backend-module-aws/src/index.ts index 212308edaa..804fd97e91 100644 --- a/plugins/catalog-backend-module-aws/src/index.ts +++ b/plugins/catalog-backend-module-aws/src/index.ts @@ -22,4 +22,5 @@ export * from './processors'; export * from './providers'; +export { awsS3EntityProviderCatalogModule } from './service/AwsS3EntityProviderCatalogModule'; export * from './types'; diff --git a/plugins/catalog-backend-module-aws/src/service/AwsS3EntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-aws/src/service/AwsS3EntityProviderCatalogModule.test.ts new file mode 100644 index 0000000000..3167291bca --- /dev/null +++ b/plugins/catalog-backend-module-aws/src/service/AwsS3EntityProviderCatalogModule.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { + PluginTaskScheduler, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { startTestBackend } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { Duration } from 'luxon'; +import { awsS3EntityProviderCatalogModule } from './AwsS3EntityProviderCatalogModule'; +import { AwsS3EntityProvider } from '../providers'; + +describe('awsS3EntityProviderCatalogModule', () => { + it('should register provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = { + createScheduledTaskRunner: (schedule: TaskScheduleDefinition) => { + usedSchedule = schedule; + return runner; + }, + } as unknown as PluginTaskScheduler; + + const config = new ConfigReader({ + catalog: { + providers: { + awsS3: { + bucketName: 'module-test', + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }); + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + services: [ + [configServiceRef, config], + [loggerServiceRef, getVoidLogger()], + [schedulerServiceRef, scheduler], + ], + features: [awsS3EntityProviderCatalogModule()], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'awsS3-provider:default', + ); + expect(runner).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/catalog-backend-module-aws/src/service/AwsS3EntityProviderCatalogModule.ts b/plugins/catalog-backend-module-aws/src/service/AwsS3EntityProviderCatalogModule.ts new file mode 100644 index 0000000000..79269ad53a --- /dev/null +++ b/plugins/catalog-backend-module-aws/src/service/AwsS3EntityProviderCatalogModule.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + createBackendModule, + loggerToWinstonLogger, + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { AwsS3EntityProvider } from '../providers'; + +/** + * Registers the AwsS3EntityProvider with the catalog processing extension point. + * + * @alpha + */ +export const awsS3EntityProviderCatalogModule = createBackendModule({ + pluginId: 'catalog', + moduleId: 'awsS3EntityProvider', + register(env) { + env.registerInit({ + deps: { + config: configServiceRef, + catalog: catalogProcessingExtensionPoint, + logger: loggerServiceRef, + scheduler: schedulerServiceRef, + }, + async init({ config, catalog, logger, scheduler }) { + catalog.addEntityProvider( + AwsS3EntityProvider.fromConfig(config, { + logger: loggerToWinstonLogger(logger), + scheduler, + }), + ); + }, + }); + }, +}); diff --git a/yarn.lock b/yarn.lock index 58cf42441d..2bd6f7bfdd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4438,13 +4438,16 @@ __metadata: resolution: "@backstage/plugin-catalog-backend-module-aws@workspace:plugins/catalog-backend-module-aws" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 aws-sdk: ^2.840.0 From 87ff05892d3fe235dedb6ea15b26e66bcd0110f5 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 00:33:18 +0200 Subject: [PATCH 03/11] feat(catalog/azure): Add option to configure schedule via `app-config.yaml` Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/clean-planets-rhyme.md | 8 ++ docs/integrations/azure/discovery.md | 33 ++++-- .../api-report.md | 4 +- .../catalog-backend-module-azure/config.d.ts | 11 +- .../catalog-backend-module-azure/package.json | 3 +- .../AzureDevOpsEntityProvider.test.ts | 106 +++++++++++++++++- .../providers/AzureDevOpsEntityProvider.ts | 29 +++-- .../src/providers/config.test.ts | 26 ++++- .../src/providers/config.ts | 6 + .../src/providers/types.ts | 3 + yarn.lock | 1 + 11 files changed, 204 insertions(+), 26 deletions(-) create mode 100644 .changeset/clean-planets-rhyme.md diff --git a/.changeset/clean-planets-rhyme.md b/.changeset/clean-planets-rhyme.md new file mode 100644 index 0000000000..a6878b491b --- /dev/null +++ b/.changeset/clean-planets-rhyme.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-azure': patch +--- + +`AzureDevOpsEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/azure/discovery diff --git a/docs/integrations/azure/discovery.md b/docs/integrations/azure/discovery.md index 6a420462c0..2156f60814 100644 --- a/docs/integrations/azure/discovery.md +++ b/docs/integrations/azure/discovery.md @@ -42,12 +42,17 @@ catalog: project: myproject repository: service-* # this will match all repos starting with service-* path: /catalog-info.yaml + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } anotherProviderId: # another identifier organization: myorg project: myproject repository: '*' # this will match all repos path: /src/*/catalog-info.yaml # this will search for files deep inside the /src folder - yetAotherProviderId: # guess, what? Another one :) + yetAnotherProviderId: # guess, what? Another one :) host: selfhostedazure.yourcompany.com organization: myorg project: myproject @@ -55,11 +60,20 @@ catalog: The parameters available are: -- `host:` Leave empty for Cloud hosted, otherwise set to your self-hosted instance host. -- `organization:` Your Organization slug (or Collection for on-premise users). Required. -- `project:` Your project slug. Required. -- `repository:` The repository name. Wildcards are supported as show on the examples above. If not set, all repositories will be searched. -- `path:` Where to find catalog-info.yaml files. Defaults to /catalog-info.yaml. +- **`host:`** _(optional)_ Leave empty for Cloud hosted, otherwise set to your self-hosted instance host. +- **`organization:`** Your Organization slug (or Collection for on-premise users). Required. +- **`project:`** Your project slug. Required. +- **`repository:`** _(optional)_ The repository name. Wildcards are supported as show on the examples above. If not set, all repositories will be searched. +- **`path:`** _(optional)_ Where to find catalog-info.yaml files. Defaults to /catalog-info.yaml. +- **`schedule`** _(optional)_: + - **`frequency`**: + How often you want the task to run. The system does its best to avoid overlapping invocations. + - **`timeout`**: + The maximum amount of time that a single task invocation can take. + - **`initialDelay`** _(optional)_: + The amount of time that should pass before the first invocation happens. + - **`scope`** _(optional)_: + `'global'` or `'local'`. Sets the scope of concurrency control. _Note:_ the path parameter follows the same rules as the search on Azure DevOps web interface. For more details visit the @@ -84,10 +98,13 @@ const builder = await CatalogBuilder.create(env); +builder.addEntityProvider( + AzureDevOpsEntityProvider.fromConfig(env.config, { + logger: env.logger, ++ // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ -+ frequency: Duration.fromObject({ minutes: 30 }), -+ timeout: Duration.fromObject({ minutes: 3 }), ++ frequency: { minutes: 30 }, ++ timeout: { minutes: 3 }, + }), ++ // optional: alternatively, use schedule ++ scheduler: env.scheduler, + }), +); ``` diff --git a/plugins/catalog-backend-module-azure/api-report.md b/plugins/catalog-backend-module-azure/api-report.md index 22f5015c9b..8fb7edeb53 100644 --- a/plugins/catalog-backend-module-azure/api-report.md +++ b/plugins/catalog-backend-module-azure/api-report.md @@ -10,6 +10,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { TaskRunner } from '@backstage/backend-tasks'; @@ -45,7 +46,8 @@ export class AzureDevOpsEntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): AzureDevOpsEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-azure/config.d.ts b/plugins/catalog-backend-module-azure/config.d.ts index 6e423c9415..940695c8ea 100644 --- a/plugins/catalog-backend-module-azure/config.d.ts +++ b/plugins/catalog-backend-module-azure/config.d.ts @@ -14,34 +14,35 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + interface AzureDevOpsConfig { /** * (Optional) The DevOps host; leave empty for `dev.azure.com`, otherwise set to your self-hosted instance host. - * @visibility backend */ host: string; /** * (Required) Your organization slug. - * @visibility backend */ organization: string; /** * (Required) Your project slug. - * @visibility backend */ project: string; /** * (Optional) The repository name. Wildcards are supported as show on the examples above. * If not set, all repositories will be searched. - * @visibility backend */ repository?: string; /** * (Optional) Where to find catalog-info.yaml files. Wildcards are supported. * If not set, defaults to /catalog-info.yaml. - * @visibility backend */ path?: string; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } export interface Config { diff --git a/plugins/catalog-backend-module-azure/package.json b/plugins/catalog-backend-module-azure/package.json index 0385655259..00dbc0d744 100644 --- a/plugins/catalog-backend-module-azure/package.json +++ b/plugins/catalog-backend-module-azure/package.json @@ -49,7 +49,8 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", - "@types/lodash": "^4.14.151" + "@types/lodash": "^4.14.151", + "luxon": "^3.0.0" }, "files": [ "dist", diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts index c57455d8d6..f7a63f259e 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts @@ -15,7 +15,11 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { CodeSearchResultItem } from '../lib'; @@ -52,6 +56,7 @@ describe('AzureDevOpsEntityProvider', () => { expectedBaseUrl: string, names: Record, integrationConfig?: object, + scheduleInConfig?: boolean, ) => { const config = new ConfigReader({ integrations: { @@ -74,9 +79,18 @@ describe('AzureDevOpsEntityProvider', () => { refresh: jest.fn(), }; + const schedulingConfig: Record = {}; + if (scheduleInConfig) { + schedulingConfig.scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + } else { + schedulingConfig.schedule = schedule; + } + const provider = AzureDevOpsEntityProvider.fromConfig(config, { + ...schedulingConfig, logger, - schedule, })[0]; expect(provider.getProviderName()).toEqual( `AzureDevOpsEntityProvider:${providerId}`, @@ -193,4 +207,92 @@ describe('AzureDevOpsEntityProvider', () => { }, ); }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + azureDevOps: { + test: { + organization: 'myorganization', + project: 'myproject', + }, + }, + }, + }, + }); + + expect(() => + AzureDevOpsEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + azureDevOps: { + test: { + organization: 'myorganization', + project: 'myproject', + }, + }, + }, + }, + }); + + expect(() => + AzureDevOpsEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for AzureDevOpsEntityProvider:test', + ); + }); + + // eslint-disable-next-line jest/expect-expect + it('single simple provider config with schedule in config', async () => { + return expectMutation( + 'allReposMultipleFiles', + { + organization: 'myorganization', + project: 'myproject', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'myrepo', + }, + }, + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'myotherrepo', + }, + }, + ], + 'https://dev.azure.com/myorganization/myproject', + { + 'myrepo?path=/catalog-info.yaml': + 'generated-87865246726bb12a8c4fb4f914443f1fbb91648c', + 'myotherrepo?path=/catalog-info.yaml': + 'generated-2deccac384c34d0dca37be0ebb4b1c8cf6913fe1', + }, + ); + }); }); diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts index aca30f367a..b824d176ee 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { AzureIntegration, ScmIntegrations } from '@backstage/integration'; import { @@ -45,11 +45,16 @@ export class AzureDevOpsEntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): AzureDevOpsEntityProvider[] { const providerConfigs = readAzureDevOpsConfigs(configRoot); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + return providerConfigs.map(providerConfig => { const integration = ScmIntegrations.fromConfig(configRoot).azure.byHost( providerConfig.host, @@ -61,11 +66,21 @@ export class AzureDevOpsEntityProvider implements EntityProvider { ); } + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for AzureDevOpsEntityProvider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + return new AzureDevOpsEntityProvider( providerConfig, integration, options.logger, - options.schedule, + taskRunner, ); }); } @@ -74,19 +89,19 @@ export class AzureDevOpsEntityProvider implements EntityProvider { private readonly config: AzureDevOpsConfig, private readonly integration: AzureIntegration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, ) { this.logger = logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-azure/src/providers/config.test.ts b/plugins/catalog-backend-module-azure/src/providers/config.test.ts index 236c9a48d4..a09079f1c3 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readAzureDevOpsConfigs } from './config'; describe('readAzureDevOpsConfigs', () => { @@ -33,17 +34,27 @@ describe('readAzureDevOpsConfigs', () => { project: 'myproject', repository: 'service-*', }; + const provider4 = { + organization: 'mycompany', + project: 'myproject', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }; const config = { catalog: { providers: { - azureDevOps: { provider1, provider2, provider3 }, + azureDevOps: { provider1, provider2, provider3, provider4 }, }, }, }; const actual = readAzureDevOpsConfigs(new ConfigReader(config)); - expect(actual).toHaveLength(3); + expect(actual).toHaveLength(4); expect(actual[0]).toEqual({ ...provider1, path: '/catalog-info.yaml', @@ -63,5 +74,16 @@ describe('readAzureDevOpsConfigs', () => { path: '/catalog-info.yaml', id: 'provider3', }); + expect(actual[3]).toEqual({ + ...provider4, + host: 'dev.azure.com', + path: '/catalog-info.yaml', + repository: '*', + id: 'provider4', + schedule: { + ...provider4.schedule, + frequency: Duration.fromISO(provider4.schedule.frequency), + }, + }); }); }); diff --git a/plugins/catalog-backend-module-azure/src/providers/config.ts b/plugins/catalog-backend-module-azure/src/providers/config.ts index 1d143fc882..c84c242488 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { AzureDevOpsConfig } from './types'; @@ -42,6 +43,10 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig { const repository = config.getOptionalString('repository') || '*'; const path = config.getOptionalString('path') || '/catalog-info.yaml'; + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, host, @@ -49,5 +54,6 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig { project, repository, path, + schedule, }; } diff --git a/plugins/catalog-backend-module-azure/src/providers/types.ts b/plugins/catalog-backend-module-azure/src/providers/types.ts index 15ea00ff13..b22a8551a5 100644 --- a/plugins/catalog-backend-module-azure/src/providers/types.ts +++ b/plugins/catalog-backend-module-azure/src/providers/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + export type AzureDevOpsConfig = { id: string; host: string; @@ -21,4 +23,5 @@ export type AzureDevOpsConfig = { project: string; repository: string; path: string; + schedule?: TaskScheduleDefinition; }; diff --git a/yarn.lock b/yarn.lock index 2bd6f7bfdd..23830a549a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4477,6 +4477,7 @@ __metadata: "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 lodash: ^4.17.21 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 uuid: ^8.0.0 From 0ca399b31bac18bde801ab58c2b904afb59e74bd Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 00:46:21 +0200 Subject: [PATCH 04/11] feat(catalog/azure): Add backend plugin Add `azureDevOpsEntityProviderCatalogModule` (new backend-plugin-api, alpha). Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/itchy-paws-protect.md | 5 ++ .../api-report.md | 6 ++ .../catalog-backend-module-azure/package.json | 14 +-- .../catalog-backend-module-azure/src/index.ts | 1 + ...eDevOpsEntityProviderCatalogModule.test.ts | 87 +++++++++++++++++++ .../AzureDevOpsEntityProviderCatalogModule.ts | 53 +++++++++++ yarn.lock | 2 + 7 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 .changeset/itchy-paws-protect.md create mode 100644 plugins/catalog-backend-module-azure/src/service/AzureDevOpsEntityProviderCatalogModule.test.ts create mode 100644 plugins/catalog-backend-module-azure/src/service/AzureDevOpsEntityProviderCatalogModule.ts diff --git a/.changeset/itchy-paws-protect.md b/.changeset/itchy-paws-protect.md new file mode 100644 index 0000000000..a0c272e5e4 --- /dev/null +++ b/.changeset/itchy-paws-protect.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-azure': patch +--- + +Add `azureDevOpsEntityProviderCatalogModule` (new backend-plugin-api, alpha). diff --git a/plugins/catalog-backend-module-azure/api-report.md b/plugins/catalog-backend-module-azure/api-report.md index 8fb7edeb53..84ffaf39e9 100644 --- a/plugins/catalog-backend-module-azure/api-report.md +++ b/plugins/catalog-backend-module-azure/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; @@ -55,4 +56,9 @@ export class AzureDevOpsEntityProvider implements EntityProvider { // (undocumented) refresh(logger: Logger): Promise; } + +// @alpha +export const azureDevOpsEntityProviderCatalogModule: ( + options?: undefined, +) => BackendFeature; ``` diff --git a/plugins/catalog-backend-module-azure/package.json b/plugins/catalog-backend-module-azure/package.json index 00dbc0d744..c8c768b1db 100644 --- a/plugins/catalog-backend-module-azure/package.json +++ b/plugins/catalog-backend-module-azure/package.json @@ -7,6 +7,7 @@ "license": "Apache-2.0", "publishConfig": { "access": "public", + "alphaTypes": "dist/index.alpha.d.ts", "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, @@ -23,22 +24,24 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", + "start": "backstage-cli package start", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", - "clean": "backstage-cli package clean", - "start": "backstage-cli package start" + "clean": "backstage-cli package clean" }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", "@backstage/types": "workspace:^", "lodash": "^4.17.21", "msw": "^0.47.0", @@ -53,8 +56,9 @@ "luxon": "^3.0.0" }, "files": [ - "dist", - "config.d.ts" + "alpha", + "config.d.ts", + "dist" ], "configSchema": "config.d.ts" } diff --git a/plugins/catalog-backend-module-azure/src/index.ts b/plugins/catalog-backend-module-azure/src/index.ts index a71c40ee6c..aa3a94fda2 100644 --- a/plugins/catalog-backend-module-azure/src/index.ts +++ b/plugins/catalog-backend-module-azure/src/index.ts @@ -22,3 +22,4 @@ export { AzureDevOpsDiscoveryProcessor } from './processors'; export { AzureDevOpsEntityProvider } from './providers'; +export { azureDevOpsEntityProviderCatalogModule } from './service/AzureDevOpsEntityProviderCatalogModule'; diff --git a/plugins/catalog-backend-module-azure/src/service/AzureDevOpsEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-azure/src/service/AzureDevOpsEntityProviderCatalogModule.test.ts new file mode 100644 index 0000000000..e3a0449a8e --- /dev/null +++ b/plugins/catalog-backend-module-azure/src/service/AzureDevOpsEntityProviderCatalogModule.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { + PluginTaskScheduler, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { startTestBackend } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { Duration } from 'luxon'; +import { azureDevOpsEntityProviderCatalogModule } from './AzureDevOpsEntityProviderCatalogModule'; +import { AzureDevOpsEntityProvider } from '../providers'; + +describe('azureDevOpsEntityProviderCatalogModule', () => { + it('should register provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = { + createScheduledTaskRunner: (schedule: TaskScheduleDefinition) => { + usedSchedule = schedule; + return runner; + }, + } as unknown as PluginTaskScheduler; + + const config = new ConfigReader({ + catalog: { + providers: { + azureDevOps: { + test: { + organization: 'myorganization', + project: 'myproject', + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }); + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + services: [ + [configServiceRef, config], + [loggerServiceRef, getVoidLogger()], + [schedulerServiceRef, scheduler], + ], + features: [azureDevOpsEntityProviderCatalogModule()], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'AzureDevOpsEntityProvider:test', + ); + expect(runner).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/catalog-backend-module-azure/src/service/AzureDevOpsEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-azure/src/service/AzureDevOpsEntityProviderCatalogModule.ts new file mode 100644 index 0000000000..f3621bee23 --- /dev/null +++ b/plugins/catalog-backend-module-azure/src/service/AzureDevOpsEntityProviderCatalogModule.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + createBackendModule, + loggerToWinstonLogger, + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { AzureDevOpsEntityProvider } from '../providers'; + +/** + * Registers the AzureDevOpsEntityProvider with the catalog processing extension point. + * + * @alpha + */ +export const azureDevOpsEntityProviderCatalogModule = createBackendModule({ + pluginId: 'catalog', + moduleId: 'azureDevOpsEntityProvider', + register(env) { + env.registerInit({ + deps: { + config: configServiceRef, + catalog: catalogProcessingExtensionPoint, + logger: loggerServiceRef, + scheduler: schedulerServiceRef, + }, + async init({ config, catalog, logger, scheduler }) { + catalog.addEntityProvider( + AzureDevOpsEntityProvider.fromConfig(config, { + logger: loggerToWinstonLogger(logger), + scheduler, + }), + ); + }, + }); + }, +}); diff --git a/yarn.lock b/yarn.lock index 23830a549a..074a6fc323 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4466,6 +4466,7 @@ __metadata: resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" @@ -4474,6 +4475,7 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 lodash: ^4.17.21 From 68f7f5a857f1b47edad74dba5ccf231447e65b1b Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 10:11:29 +0200 Subject: [PATCH 05/11] feat(catalog/bitbucketServer): Add option to configure schedule via `app-config.yaml` Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/tasty-scissors-tickle.md | 8 + .../integrations/bitbucketServer/discovery.md | 36 ++-- .../api-report.md | 4 +- .../config.d.ts | 13 +- .../package.json | 1 + .../BitbucketServerEntityProvider.test.ts | 171 +++++++++++++++++- .../BitbucketServerEntityProvider.ts | 52 ++++-- ...itbucketServerEntityProviderConfig.test.ts | 27 ++- .../BitbucketServerEntityProviderConfig.ts | 10 + yarn.lock | 1 + 10 files changed, 285 insertions(+), 38 deletions(-) create mode 100644 .changeset/tasty-scissors-tickle.md diff --git a/.changeset/tasty-scissors-tickle.md b/.changeset/tasty-scissors-tickle.md new file mode 100644 index 0000000000..745172df36 --- /dev/null +++ b/.changeset/tasty-scissors-tickle.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-server': patch +--- + +`BitbucketServerEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/bitbucketServer/discovery diff --git a/docs/integrations/bitbucketServer/discovery.md b/docs/integrations/bitbucketServer/discovery.md index 57e52d1dae..c4288b33ec 100644 --- a/docs/integrations/bitbucketServer/discovery.md +++ b/docs/integrations/bitbucketServer/discovery.md @@ -37,10 +37,13 @@ And then add the entity provider to your catalog builder: + builder.addEntityProvider( + BitbucketServerEntityProvider.fromConfig(env.config, { + logger: env.logger, ++ // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), ++ // optional: alternatively, use schedule ++ scheduler: env.scheduler, + }), + ); @@ -66,19 +69,33 @@ catalog: filters: # optional projectKey: '^apis-.*$' # optional; RegExp repoSlug: '^service-.*$' # optional; RegExp + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` -- **host**: +- **`host`**: The host of the Bitbucket Server instance, **note**: the host needs to registered as an integration as well, see [location](locations.md). - **`catalogPath`** _(optional)_: Default: `/catalog-info.yaml`. Path where to look for `catalog-info.yaml` files. When started with `/`, it is an absolute path from the repo root. -- **filters** _(optional)_: +- **`filters`** _(optional)_: - **`projectKey`** _(optional)_: Regular expression used to filter results based on the project key. - - **repoSlug** _(optional)_: + - **`repoSlug`** _(optional)_: Regular expression used to filter results based on the repo slug. +- **`schedule`** _(optional)_: + - **`frequency`**: + How often you want the task to run. The system does its best to avoid overlapping invocations. + - **`timeout`**: + The maximum amount of time that a single task invocation can take. + - **`initialDelay`** _(optional)_: + The amount of time that should pass before the first invocation happens. + - **`scope`** _(optional)_: + `'global'` or `'local'`. Sets the scope of concurrency control. ## Custom location processing @@ -93,18 +110,15 @@ repository. ```typescript const provider = BitbucketServerEntityProvider.fromConfig(env.config, { logger: env.logger, - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 30 }, - timeout: { minutes: 3 }, - }), + schedule: env.scheduler, parser: async function* customLocationParser(options: { - location: LocationSpec, - client: BitbucketServerClient, + location: LocationSpec; + client: BitbucketServerClient; }) { // Custom logic for interpreting the matching repository // See defaultBitbucketServerLocationParser for an example - } -); + }, +}); ``` ## Alternative diff --git a/plugins/catalog-backend-module-bitbucket-server/api-report.md b/plugins/catalog-backend-module-bitbucket-server/api-report.md index 7d95e148a3..6c840883af 100644 --- a/plugins/catalog-backend-module-bitbucket-server/api-report.md +++ b/plugins/catalog-backend-module-bitbucket-server/api-report.md @@ -10,6 +10,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Response as Response_2 } from 'node-fetch'; import { TaskRunner } from '@backstage/backend-tasks'; @@ -55,8 +56,9 @@ export class BitbucketServerEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; parser?: BitbucketServerLocationParser; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): BitbucketServerEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-bitbucket-server/config.d.ts b/plugins/catalog-backend-module-bitbucket-server/config.d.ts index d8a09c92a4..67f694d419 100644 --- a/plugins/catalog-backend-module-bitbucket-server/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-server/config.d.ts @@ -14,11 +14,10 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { catalog?: { - /** - * List of provider-specific options and attributes - */ providers?: { /** * BitbucketServerEntityProvider configuration @@ -48,6 +47,10 @@ export interface Config { */ projectKey?: RegExp; }; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } | Record< string, @@ -73,6 +76,10 @@ export interface Config { */ projectKey?: RegExp; }; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-bitbucket-server/package.json b/plugins/catalog-backend-module-bitbucket-server/package.json index 2490b79ffa..021b531cea 100644 --- a/plugins/catalog-backend-module-bitbucket-server/package.json +++ b/plugins/catalog-backend-module-bitbucket-server/package.json @@ -46,6 +46,7 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", + "luxon": "^3.0.0", "msw": "^0.47.0" }, "files": [ diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts index 39689b6c1d..20299e3452 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts @@ -15,14 +15,18 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; -import { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import { BitbucketServerPagedResponse } from '../lib/BitbucketServerClient'; +import { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider'; +import { BitbucketServerPagedResponse } from '../lib'; class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -367,4 +371,163 @@ describe('BitbucketServerEntityProvider', () => { entities: expectedEntities, }); }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + host: 'bitbucket.mycompany.com', + }, + }, + }, + integrations: { + bitbucketServer: [ + { + host: 'bitbucket.mycompany.com', + }, + ], + }, + }); + + expect(() => + BitbucketServerEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + host: 'bitbucket.mycompany.com', + }, + }, + }, + integrations: { + bitbucketServer: [ + { + host: 'bitbucket.mycompany.com', + }, + ], + }, + }); + + expect(() => + BitbucketServerEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for bitbucketServer-provider:default', + ); + }); + + it('apply full update with schedule in config', async () => { + const host = 'bitbucket.mycompany.com'; + const config = new ConfigReader({ + integrations: { + bitbucketServer: [ + { + host: host, + }, + ], + }, + catalog: { + providers: { + bitbucketServer: { + mainProvider: { + host: host, + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = BitbucketServerEntityProvider.fromConfig(config, { + logger, + scheduler, + })[0]; + expect(provider.getProviderName()).toEqual( + 'bitbucketServer-provider:mainProvider', + ); + + setupStubs( + [ + { key: 'project-test', repos: ['repo-test'] }, + { key: 'other-project', repos: ['other-repo'] }, + ], + `https://${host}`, + ); + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('bitbucketServer-provider:mainProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`, + 'backstage.io/managed-by-origin-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`, + }, + name: 'generated-77f4323822420990f8c3e3c981d38c2dec4ae3a6', + }, + spec: { + presence: 'optional', + target: `https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`, + type: 'url', + }, + }, + locationKey: 'bitbucketServer-provider:mainProvider', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + 'backstage.io/managed-by-origin-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + }, + name: 'generated-d8d4944c30c2906dfee172ddda9537f9893b2c0f', + }, + spec: { + presence: 'optional', + target: `https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + type: 'url', + }, + }, + locationKey: 'bitbucketServer-provider:mainProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); }); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts index 92cecf642a..447b71ee58 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts @@ -14,29 +14,29 @@ * limitations under the License. */ +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; +import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { InputError } from '@backstage/errors'; +import { + BitbucketServerIntegration, + ScmIntegrations, +} from '@backstage/integration'; import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; -import { Config } from '@backstage/config'; -import { TaskRunner } from '@backstage/backend-tasks'; import * as uuid from 'uuid'; -import { - BitbucketServerLocationParser, - defaultBitbucketServerLocationParser, -} from './BitbucketServerLocationParser'; -import { - BitbucketServerIntegration, - ScmIntegrations, -} from '@backstage/integration'; import { BitbucketServerClient, paginated } from '../lib'; -import { InputError } from '@backstage/errors'; import { BitbucketServerEntityProviderConfig, readProviderConfigs, } from './BitbucketServerEntityProviderConfig'; -import { Entity } from '@backstage/catalog-model'; +import { + BitbucketServerLocationParser, + defaultBitbucketServerLocationParser, +} from './BitbucketServerLocationParser'; /** * Discovers catalog files located in Bitbucket Server. @@ -58,12 +58,17 @@ export class BitbucketServerEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; parser?: BitbucketServerLocationParser; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): BitbucketServerEntityProvider[] { const integrations = ScmIntegrations.fromConfig(config); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + return readProviderConfigs(config).map(providerConfig => { const integration = integrations.bitbucketServer.byHost( providerConfig.host, @@ -73,11 +78,22 @@ export class BitbucketServerEntityProvider implements EntityProvider { `No BitbucketServer integration found that matches host ${providerConfig.host}`, ); } + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for bitbucketServer-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + return new BitbucketServerEntityProvider( providerConfig, integration, options.logger, - options.schedule, + taskRunner, options.parser, ); }); @@ -87,7 +103,7 @@ export class BitbucketServerEntityProvider implements EntityProvider { config: BitbucketServerEntityProviderConfig, integration: BitbucketServerIntegration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, parser?: BitbucketServerLocationParser, ) { this.integration = integration; @@ -96,13 +112,13 @@ export class BitbucketServerEntityProvider implements EntityProvider { this.logger = logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts index 712b26cbb0..af8639566f 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readProviderConfigs } from './BitbucketServerEntityProviderConfig'; describe('readProviderConfigs', () => { @@ -63,13 +64,22 @@ describe('readProviderConfigs', () => { host: 'bitbucket2.mycompany.com', catalogPath: 'custom/path/catalog-info.yaml', }, + thirdProvider: { + host: 'bitbucket3.mycompany.com', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(2); + expect(providerConfigs).toHaveLength(3); expect(providerConfigs[0]).toEqual({ id: 'mainProvider', catalogPath: '/catalog-info.yaml', @@ -88,6 +98,21 @@ describe('readProviderConfigs', () => { repoSlug: undefined, }, }); + expect(providerConfigs[2]).toEqual({ + id: 'thirdProvider', + catalogPath: '/catalog-info.yaml', + host: 'bitbucket3.mycompany.com', + filters: { + projectKey: undefined, + repoSlug: undefined, + }, + schedule: { + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 3, + }, + }, + }); }); it('single provider config with filters', () => { diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts index 8a3decab43..b2ed7a8556 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts @@ -14,6 +14,10 @@ * limitations under the License. */ +import { + readTaskScheduleDefinitionFromConfig, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; @@ -27,6 +31,7 @@ export type BitbucketServerEntityProviderConfig = { projectKey?: RegExp; repoSlug?: RegExp; }; + schedule?: TaskScheduleDefinition; }; export function readProviderConfigs( @@ -60,6 +65,10 @@ function readProviderConfig( const projectKeyPattern = config.getOptionalString('filters.projectKey'); const repoSlugPattern = config.getOptionalString('filters.repoSlug'); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, host, @@ -68,5 +77,6 @@ function readProviderConfig( projectKey: projectKeyPattern ? new RegExp(projectKeyPattern) : undefined, repoSlug: repoSlugPattern ? new RegExp(repoSlugPattern) : undefined, }, + schedule, }; } diff --git a/yarn.lock b/yarn.lock index 074a6fc323..b90c1a515d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4522,6 +4522,7 @@ __metadata: "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" "@types/node-fetch": ^2.5.12 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 uuid: ^8.0.0 From cd48ed8370c3b46e7769780db032b97987633125 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 10:28:48 +0200 Subject: [PATCH 06/11] feat(catalog/bitbucketServer): Add backend plugin Add `bitbucketServerEntityProviderCatalogModule` (new backend-plugin-api, alpha). Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/two-oranges-joke.md | 5 + .../api-report.md | 6 ++ .../package.json | 14 ++- .../src/index.ts | 1 + ...tServerEntityProviderCatalogModule.test.ts | 91 +++++++++++++++++++ ...bucketServerEntityProviderCatalogModule.ts | 52 +++++++++++ yarn.lock | 2 + 7 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 .changeset/two-oranges-joke.md create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/service/BitbucketServerEntityProviderCatalogModule.test.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/service/BitbucketServerEntityProviderCatalogModule.ts diff --git a/.changeset/two-oranges-joke.md b/.changeset/two-oranges-joke.md new file mode 100644 index 0000000000..7f116e8ab0 --- /dev/null +++ b/.changeset/two-oranges-joke.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-server': patch +--- + +Add `bitbucketServerEntityProviderCatalogModule` (new backend-plugin-api, alpha). diff --git a/plugins/catalog-backend-module-bitbucket-server/api-report.md b/plugins/catalog-backend-module-bitbucket-server/api-report.md index 6c840883af..a38a4d8c35 100644 --- a/plugins/catalog-backend-module-bitbucket-server/api-report.md +++ b/plugins/catalog-backend-module-bitbucket-server/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; import { BitbucketServerIntegrationConfig } from '@backstage/integration'; import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; @@ -67,6 +68,11 @@ export class BitbucketServerEntityProvider implements EntityProvider { refresh(logger: Logger): Promise; } +// @alpha (undocumented) +export const bitbucketServerEntityProviderCatalogModule: ( + options?: undefined, +) => BackendFeature; + // @public (undocumented) export type BitbucketServerListOptions = { [key: string]: number | undefined; diff --git a/plugins/catalog-backend-module-bitbucket-server/package.json b/plugins/catalog-backend-module-bitbucket-server/package.json index 021b531cea..9d2621ea40 100644 --- a/plugins/catalog-backend-module-bitbucket-server/package.json +++ b/plugins/catalog-backend-module-bitbucket-server/package.json @@ -6,6 +6,7 @@ "license": "Apache-2.0", "publishConfig": { "access": "public", + "alphaTypes": "dist/index.alpha.d.ts", "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, @@ -23,21 +24,23 @@ ], "scripts": { "start": "backstage-cli package start", - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", - "clean": "backstage-cli package clean", "prepack": "backstage-cli package prepack", - "postpack": "backstage-cli package postpack" + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean" }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", "@types/node-fetch": "^2.5.12", "node-fetch": "^2.6.7", "uuid": "^8.0.0", @@ -50,8 +53,9 @@ "msw": "^0.47.0" }, "files": [ - "dist", - "config.d.ts" + "alpha", + "config.d.ts", + "dist" ], "configSchema": "config.d.ts" } diff --git a/plugins/catalog-backend-module-bitbucket-server/src/index.ts b/plugins/catalog-backend-module-bitbucket-server/src/index.ts index 34ac3a9966..f139078cfb 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/index.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/index.ts @@ -29,3 +29,4 @@ export type { } from './lib'; export { BitbucketServerEntityProvider } from './providers'; export type { BitbucketServerLocationParser } from './providers'; +export { bitbucketServerEntityProviderCatalogModule } from './service/BitbucketServerEntityProviderCatalogModule'; diff --git a/plugins/catalog-backend-module-bitbucket-server/src/service/BitbucketServerEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/service/BitbucketServerEntityProviderCatalogModule.test.ts new file mode 100644 index 0000000000..d2ce4dbd3d --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/service/BitbucketServerEntityProviderCatalogModule.test.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { + PluginTaskScheduler, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { startTestBackend } from '@backstage/backend-test-utils'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { bitbucketServerEntityProviderCatalogModule } from './BitbucketServerEntityProviderCatalogModule'; +import { Duration } from 'luxon'; +import { BitbucketServerEntityProvider } from '../providers'; + +describe('bitbucketServerEntityProviderCatalogModule', () => { + it('should register provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = { + createScheduledTaskRunner: (schedule: TaskScheduleDefinition) => { + usedSchedule = schedule; + return runner; + }, + } as unknown as PluginTaskScheduler; + + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + host: 'bitbucket.mycompany.com', + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + integrations: { + bitbucketServer: [ + { + host: 'bitbucket.mycompany.com', + }, + ], + }, + }); + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + services: [ + [configServiceRef, config], + [loggerServiceRef, getVoidLogger()], + [schedulerServiceRef, scheduler], + ], + features: [bitbucketServerEntityProviderCatalogModule()], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'bitbucketServer-provider:default', + ); + expect(runner).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/service/BitbucketServerEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-bitbucket-server/src/service/BitbucketServerEntityProviderCatalogModule.ts new file mode 100644 index 0000000000..e89eee7b45 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/service/BitbucketServerEntityProviderCatalogModule.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + configServiceRef, + createBackendModule, + loggerServiceRef, + loggerToWinstonLogger, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { BitbucketServerEntityProvider } from '../providers'; + +/** + * @alpha + */ +export const bitbucketServerEntityProviderCatalogModule = createBackendModule({ + pluginId: 'catalog', + moduleId: 'bitbucketServerEntityProvider', + register(env) { + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + config: configServiceRef, + logger: loggerServiceRef, + scheduler: schedulerServiceRef, + }, + async init({ catalog, config, logger, scheduler }) { + const winstonLogger = loggerToWinstonLogger(logger); + const providers = BitbucketServerEntityProvider.fromConfig(config, { + logger: winstonLogger, + scheduler, + }); + + catalog.addEntityProvider(providers); + }, + }); + }, +}); diff --git a/yarn.lock b/yarn.lock index b90c1a515d..1551240610 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4513,6 +4513,7 @@ __metadata: resolution: "@backstage/plugin-catalog-backend-module-bitbucket-server@workspace:plugins/catalog-backend-module-bitbucket-server" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" @@ -4521,6 +4522,7 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" "@types/node-fetch": ^2.5.12 luxon: ^3.0.0 msw: ^0.47.0 From 134b69f478118657ff07ae8b5c80576566ee8f02 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 11:11:07 +0200 Subject: [PATCH 07/11] feat(catalog/gerrit): Add option to configure schedule via `app-config.yaml` Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/ten-pens-draw.md | 8 ++ docs/integrations/gerrit/discovery.md | 16 ++- .../api-report.md | 4 +- .../package.json | 3 +- .../providers/GerritEntityProvider.test.ts | 101 +++++++++++++++++- .../src/providers/GerritEntityProvider.ts | 30 ++++-- .../src/providers/config.test.ts | 24 ++++- .../src/providers/config.ts | 6 ++ .../src/providers/types.ts | 3 + yarn.lock | 1 + 10 files changed, 178 insertions(+), 18 deletions(-) create mode 100644 .changeset/ten-pens-draw.md diff --git a/.changeset/ten-pens-draw.md b/.changeset/ten-pens-draw.md new file mode 100644 index 0000000000..be35a6f3b0 --- /dev/null +++ b/.changeset/ten-pens-draw.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-gerrit': patch +--- + +`GerritEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/gerrit/discovery diff --git a/docs/integrations/gerrit/discovery.md b/docs/integrations/gerrit/discovery.md index 04fa3dac2a..7ed05d2b3b 100644 --- a/docs/integrations/gerrit/discovery.md +++ b/docs/integrations/gerrit/discovery.md @@ -32,10 +32,13 @@ const builder = await CatalogBuilder.create(env); builder.addEntityProvider( GerritEntityProvider.fromConfig(env.config, { logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml schedule: env.scheduler.createScheduledTaskRunner({ frequency: { minutes: 30 }, timeout: { minutes: 3 }, }), + // optional: alternatively, use schedule + scheduler: env.scheduler, }), ); ``` @@ -54,6 +57,11 @@ catalog: host: gerrit-your-company.com branch: master # Optional query: 'state=ACTIVE&prefix=webapps' + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } backend: host: gerrit-your-company.com branch: master # Optional @@ -62,8 +70,8 @@ catalog: The provider configuration is composed of three parts: -- host, the host of the Gerrit integration to use. -- branch, the branch where we will look for catalog entities (defaults to "master"). -- query, this string is directly used as the argument to the "List Project" API. - Typically you will want to have some filter here to exclude projects that will +- **`host`**: the host of the Gerrit integration to use. +- **`branch`** _(optional)_: the branch where we will look for catalog entities (defaults to "master"). +- **`query`**: this string is directly used as the argument to the "List Project" API. + Typically, you will want to have some filter here to exclude projects that will never contain any catalog files. diff --git a/plugins/catalog-backend-module-gerrit/api-report.md b/plugins/catalog-backend-module-gerrit/api-report.md index 7eae94c2a8..1dee786da1 100644 --- a/plugins/catalog-backend-module-gerrit/api-report.md +++ b/plugins/catalog-backend-module-gerrit/api-report.md @@ -7,6 +7,7 @@ import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; // @public (undocumented) @@ -18,7 +19,8 @@ export class GerritEntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): GerritEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-gerrit/package.json b/plugins/catalog-backend-module-gerrit/package.json index e3665a96ae..1cebc25bc8 100644 --- a/plugins/catalog-backend-module-gerrit/package.json +++ b/plugins/catalog-backend-module-gerrit/package.json @@ -44,7 +44,8 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", - "@types/fs-extra": "^9.0.1" + "@types/fs-extra": "^9.0.1", + "luxon": "^3.0.0" }, "files": [ "dist", diff --git a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts index d6ad0ff668..5d258ee94f 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts @@ -15,14 +15,18 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { ConfigReader } from '@backstage/config'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; -import { rest } from 'msw'; import fs from 'fs-extra'; -import path from 'path'; +import { rest } from 'msw'; import { setupServer } from 'msw/node'; +import path from 'path'; import { GerritEntityProvider } from './GerritEntityProvider'; const server = setupServer(); @@ -207,4 +211,93 @@ describe('GerritEntityProvider', () => { }), ).toThrow(/No gerrit integration/); }); + + it('fail without schedule and scheduler', () => { + expect(() => + GerritEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + expect(() => + GerritEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for gerrit-provider:active-training', + ); + }); + + it('discovers projects from the api with schedule in config', async () => { + const configWithSchedule = new ConfigReader({ + catalog: { + providers: { + gerrit: { + 'active-training': { + host: 'g.com', + query: 'state=ACTIVE&prefix=training', + branch: 'main', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + }, + }, + }, + integrations: { + gerrit: [ + { + host: 'g.com', + baseUrl: 'https://g.com/gerrit', + gitilesBaseUrl: 'https:/g.com/gitiles', + }, + ], + }, + }); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + + const repoBuffer = fs.readFileSync( + path.resolve(__dirname, '__fixtures__/listProjectsBody.txt'), + ); + const expected = getJsonFixture('expectedProviderEntities.json'); + + server.use( + rest.get('https://g.com/gerrit/projects/', (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.body(repoBuffer), + ), + ), + ); + + const provider = GerritEntityProvider.fromConfig(configWithSchedule, { + logger, + scheduler, + })[0]; + expect(provider.getProviderName()).toEqual( + 'gerrit-provider:active-training', + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('gerrit-provider:active-training:refresh'); + await (taskDef.fn as () => Promise)(); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith( + expected, + ); + }); }); diff --git a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.ts b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.ts index beb4052073..ec08dbe4be 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { InputError } from '@backstage/errors'; import { @@ -49,9 +49,14 @@ export class GerritEntityProvider implements EntityProvider { configRoot: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): GerritEntityProvider[] { + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + const providerConfigs = readGerritConfigs(configRoot); const integrations = ScmIntegrations.fromConfig(configRoot).gerrit; const providers: GerritEntityProvider[] = []; @@ -63,12 +68,23 @@ export class GerritEntityProvider implements EntityProvider { `No gerrit integration found that matches host ${providerConfig.host}`, ); } + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for gerrit-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + providers.push( new GerritEntityProvider( providerConfig, integration, options.logger, - options.schedule, + taskRunner, ), ); }); @@ -79,14 +95,14 @@ export class GerritEntityProvider implements EntityProvider { config: GerritProviderConfig, integration: GerritIntegration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, ) { this.config = config; this.integration = integration; this.logger = logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); } getProviderName(): string { @@ -98,10 +114,10 @@ export class GerritEntityProvider implements EntityProvider { await this.scheduleFn(); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-gerrit/src/providers/config.test.ts b/plugins/catalog-backend-module-gerrit/src/providers/config.test.ts index 0422b4e4ea..ecf68d9f3c 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readGerritConfigs } from './config'; describe('readGerritConfigs', () => { @@ -29,12 +30,24 @@ describe('readGerritConfigs', () => { query: 'state=ACTIVE', branch: 'main', }; + const provider3 = { + host: 'gerrit1.com', + query: 'state=ACTIVE', + branch: 'main', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }; const config = { catalog: { providers: { gerrit: { 'active-g1': provider1, 'active-g2': provider2, + 'active-g3': provider3, }, }, }, @@ -42,10 +55,19 @@ describe('readGerritConfigs', () => { const actual = readGerritConfigs(new ConfigReader(config)); - expect(actual).toHaveLength(2); + expect(actual).toHaveLength(3); expect(actual[0]).toEqual({ ...provider1, id: 'active-g1' }); expect(actual[1]).toEqual({ ...provider2, id: 'active-g2' }); + expect(actual[2]).toEqual({ + ...provider3, + id: 'active-g3', + schedule: { + ...provider3.schedule, + frequency: Duration.fromISO(provider3.schedule.frequency), + }, + }); }); + it('provides default values', () => { const provider = { host: 'gerrit1.com', diff --git a/plugins/catalog-backend-module-gerrit/src/providers/config.ts b/plugins/catalog-backend-module-gerrit/src/providers/config.ts index 7a28ddd8ea..dd7ee9ec43 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/config.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/config.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GerritProviderConfig } from './types'; @@ -22,11 +23,16 @@ function readGerritConfig(id: string, config: Config): GerritProviderConfig { const host = config.getString('host'); const query = config.getString('query'); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { branch, host, id, query, + schedule, }; } diff --git a/plugins/catalog-backend-module-gerrit/src/providers/types.ts b/plugins/catalog-backend-module-gerrit/src/providers/types.ts index 1e96bd1ee4..4a8329df86 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/types.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + export type GerritProjectInfo = { id: string; name: string; @@ -28,4 +30,5 @@ export type GerritProviderConfig = { query: string; id: string; branch?: string; + schedule?: TaskScheduleDefinition; }; diff --git a/yarn.lock b/yarn.lock index 1551240610..bfb1485395 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4569,6 +4569,7 @@ __metadata: "@backstage/plugin-catalog-backend": "workspace:^" "@types/fs-extra": ^9.0.1 fs-extra: 10.1.0 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 uuid: ^8.0.0 From 4fba50f5d431d045188419ebadd6ea02fb32ca87 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 11:29:54 +0200 Subject: [PATCH 08/11] feat(catalog/gerrit): Add backend plugin Add `gerritEntityProviderCatalogModule` (new backend-plugin-api, alpha). Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/metal-dogs-swim.md | 5 + .../api-report.md | 6 ++ .../package.json | 9 +- .../src/index.ts | 1 + .../GerritEntityProviderCatalogModule.test.ts | 97 +++++++++++++++++++ .../GerritEntityProviderCatalogModule.ts | 52 ++++++++++ yarn.lock | 2 + 7 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 .changeset/metal-dogs-swim.md create mode 100644 plugins/catalog-backend-module-gerrit/src/service/GerritEntityProviderCatalogModule.test.ts create mode 100644 plugins/catalog-backend-module-gerrit/src/service/GerritEntityProviderCatalogModule.ts diff --git a/.changeset/metal-dogs-swim.md b/.changeset/metal-dogs-swim.md new file mode 100644 index 0000000000..d97c7077f2 --- /dev/null +++ b/.changeset/metal-dogs-swim.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gerrit': patch +--- + +Add `gerritEntityProviderCatalogModule` (new backend-plugin-api, alpha). diff --git a/plugins/catalog-backend-module-gerrit/api-report.md b/plugins/catalog-backend-module-gerrit/api-report.md index 1dee786da1..45847ea93d 100644 --- a/plugins/catalog-backend-module-gerrit/api-report.md +++ b/plugins/catalog-backend-module-gerrit/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; @@ -29,5 +30,10 @@ export class GerritEntityProvider implements EntityProvider { refresh(logger: Logger): Promise; } +// @alpha (undocumented) +export const gerritEntityProviderCatalogModule: ( + options?: undefined, +) => BackendFeature; + // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/catalog-backend-module-gerrit/package.json b/plugins/catalog-backend-module-gerrit/package.json index 1cebc25bc8..43be593478 100644 --- a/plugins/catalog-backend-module-gerrit/package.json +++ b/plugins/catalog-backend-module-gerrit/package.json @@ -6,6 +6,7 @@ "license": "Apache-2.0", "publishConfig": { "access": "public", + "alphaTypes": "dist/index.alpha.d.ts", "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, @@ -20,21 +21,23 @@ }, "scripts": { "start": "backstage-cli package start", - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", - "clean": "backstage-cli package clean", "prepack": "backstage-cli package prepack", - "postpack": "backstage-cli package postpack" + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean" }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", "fs-extra": "10.1.0", "msw": "^0.47.0", "node-fetch": "^2.6.7", diff --git a/plugins/catalog-backend-module-gerrit/src/index.ts b/plugins/catalog-backend-module-gerrit/src/index.ts index 133772d4b1..38ede69926 100644 --- a/plugins/catalog-backend-module-gerrit/src/index.ts +++ b/plugins/catalog-backend-module-gerrit/src/index.ts @@ -15,3 +15,4 @@ */ export { GerritEntityProvider } from './providers/GerritEntityProvider'; +export { gerritEntityProviderCatalogModule } from './service/GerritEntityProviderCatalogModule'; diff --git a/plugins/catalog-backend-module-gerrit/src/service/GerritEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-gerrit/src/service/GerritEntityProviderCatalogModule.test.ts new file mode 100644 index 0000000000..53e012209b --- /dev/null +++ b/plugins/catalog-backend-module-gerrit/src/service/GerritEntityProviderCatalogModule.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { + PluginTaskScheduler, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { startTestBackend } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { Duration } from 'luxon'; +import { gerritEntityProviderCatalogModule } from './GerritEntityProviderCatalogModule'; +import { GerritEntityProvider } from '../providers/GerritEntityProvider'; + +describe('gerritEntityProviderCatalogModule', () => { + it('should register provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = { + createScheduledTaskRunner: (schedule: TaskScheduleDefinition) => { + usedSchedule = schedule; + return runner; + }, + } as unknown as PluginTaskScheduler; + + const config = new ConfigReader({ + catalog: { + providers: { + gerrit: { + test: { + host: 'g.com', + query: 'state=ACTIVE&prefix=training', + branch: 'main', + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + integrations: { + gerrit: [ + { + host: 'g.com', + baseUrl: 'https://g.com/gerrit', + gitilesBaseUrl: 'https:/g.com/gitiles', + }, + ], + }, + }); + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + services: [ + [configServiceRef, config], + [loggerServiceRef, getVoidLogger()], + [schedulerServiceRef, scheduler], + ], + features: [gerritEntityProviderCatalogModule()], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'gerrit-provider:test', + ); + expect(runner).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/catalog-backend-module-gerrit/src/service/GerritEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-gerrit/src/service/GerritEntityProviderCatalogModule.ts new file mode 100644 index 0000000000..32d582e397 --- /dev/null +++ b/plugins/catalog-backend-module-gerrit/src/service/GerritEntityProviderCatalogModule.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + configServiceRef, + createBackendModule, + loggerServiceRef, + loggerToWinstonLogger, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { GerritEntityProvider } from '../providers/GerritEntityProvider'; + +/** + * @alpha + */ +export const gerritEntityProviderCatalogModule = createBackendModule({ + pluginId: 'catalog', + moduleId: 'gerritEntityProvider', + register(env) { + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + config: configServiceRef, + logger: loggerServiceRef, + scheduler: schedulerServiceRef, + }, + async init({ catalog, config, logger, scheduler }) { + const winstonLogger = loggerToWinstonLogger(logger); + const providers = GerritEntityProvider.fromConfig(config, { + logger: winstonLogger, + scheduler, + }); + + catalog.addEntityProvider(providers); + }, + }); + }, +}); diff --git a/yarn.lock b/yarn.lock index bfb1485395..787cb2a94a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4559,6 +4559,7 @@ __metadata: resolution: "@backstage/plugin-catalog-backend-module-gerrit@workspace:plugins/catalog-backend-module-gerrit" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" @@ -4567,6 +4568,7 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" "@types/fs-extra": ^9.0.1 fs-extra: 10.1.0 luxon: ^3.0.0 From 9b698abfd96851abae646511f1bc08aa1deef969 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 12:19:57 +0200 Subject: [PATCH 09/11] fix(catalog/gitlab): add missing configSchema to package.json Signed-off-by: Patrick Jungermann --- plugins/catalog-backend-module-gitlab/package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 510a4eb3be..90ce82b01e 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -53,6 +53,8 @@ "@types/uuid": "^8.0.0" }, "files": [ + "config.d.ts", "dist" - ] + ], + "configSchema": "config.d.ts" } From 81cedb503351d0df1449f7ffd5ee8c41973147ad Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 12:13:18 +0200 Subject: [PATCH 10/11] feat(catalog/gitlab): Add option to configure schedule via `app-config.yaml` Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/flat-kangaroos-kiss.md | 8 ++ docs/integrations/gitlab/discovery.md | 8 ++ .../api-report.md | 4 +- .../catalog-backend-module-gitlab/config.d.ts | 13 +- .../package.json | 3 +- .../src/lib/types.ts | 3 + .../GitlabDiscoveryEntityProvider.test.ts | 117 +++++++++++++++++- .../GitlabDiscoveryEntityProvider.ts | 38 ++++-- .../src/providers/config.test.ts | 45 +++++++ .../src/providers/config.ts | 6 + yarn.lock | 1 + 11 files changed, 225 insertions(+), 21 deletions(-) create mode 100644 .changeset/flat-kangaroos-kiss.md diff --git a/.changeset/flat-kangaroos-kiss.md b/.changeset/flat-kangaroos-kiss.md new file mode 100644 index 0000000000..a79c8621b1 --- /dev/null +++ b/.changeset/flat-kangaroos-kiss.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +`GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/gitlab/discovery diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index df5aadcab4..dc0b911b3d 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -25,6 +25,11 @@ catalog: group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole project will be scanned entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` projectPattern: /[\s\S]*/ # Optional. Filters found projects based on provided patter. Defaults to `/[\s\S]*/`, what means to not filter anything + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` As this provider is not one of the default providers, you will first need to install @@ -47,10 +52,13 @@ const builder = await CatalogBuilder.create(env); builder.addEntityProvider( ...GitlabDiscoveryEntityProvider.fromConfig(env.config, { logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml schedule: env.scheduler.createScheduledTaskRunner({ frequency: { minutes: 30 }, timeout: { minutes: 3 }, }), + // optional: alternatively, use schedule + scheduler: env.scheduler, }), ); ``` diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index 1009da8209..bd7b13efc3 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -10,6 +10,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; // @public @@ -21,7 +22,8 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): GitlabDiscoveryEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-gitlab/config.d.ts b/plugins/catalog-backend-module-gitlab/config.d.ts index 82f1a5db7c..ac75a04c2a 100644 --- a/plugins/catalog-backend-module-gitlab/config.d.ts +++ b/plugins/catalog-backend-module-gitlab/config.d.ts @@ -14,11 +14,10 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { catalog?: { - /** - * List of provider-specific options and attributes - */ providers?: { /** * GitlabDiscoveryEntityProvider configuration @@ -28,27 +27,27 @@ export interface Config { { /** * (Required) Gitlab's host name. - * @visibility backend */ host: string; /** * (Optional) Gitlab's group[/subgroup] where the discovery is done. * If not defined the whole project will be scanned. - * @visibility backend */ group?: string; /** * (Optional) Default branch to read the catalog-info.yaml file. * If not set, 'master' will be used. - * @visibility backend */ branch?: string; /** * (Optional) The name used for the catalog file. * If not set, 'catalog-info.yaml' will be used. - * @visibility backend */ entityFilename?: string; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 90ce82b01e..e537fa5e04 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -50,7 +50,8 @@ "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", - "@types/uuid": "^8.0.0" + "@types/uuid": "^8.0.0", + "luxon": "^3.0.0" }, "files": [ "config.d.ts", diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 8a3c33c8f2..86fe1aa385 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + export type GitlabGroupDescription = { id: number; web_url: string; @@ -36,4 +38,5 @@ export type GitlabProviderConfig = { branch: string; catalogFile: string; projectPattern: RegExp; + schedule?: TaskScheduleDefinition; }; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index 3f3c4e7f91..ae19edc48c 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -15,13 +15,17 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; -import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; +import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -338,4 +342,111 @@ describe('GitlabDiscoveryEntityProvider', () => { ], }); }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + }, + }, + }, + }); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + }, + }, + }, + }); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id', + ); + }); + + it('single simple provider config with schedule in config', async () => { + const schedule = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }); + const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index ffb123edf7..cb4629b820 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -14,14 +14,16 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; import { EntityProvider, EntityProviderConnection, + LocationSpec, + locationSpecToLocationEntity, } from '@backstage/plugin-catalog-backend'; -import { LocationSpec } from '@backstage/plugin-catalog-backend'; +import * as uuid from 'uuid'; import { Logger } from 'winston'; import { GitLabClient, @@ -30,8 +32,6 @@ import { paginated, readGitlabConfigs, } from '../lib'; -import * as uuid from 'uuid'; -import { locationSpecToLocationEntity } from '@backstage/plugin-catalog-backend'; type Result = { scanned: number; @@ -51,8 +51,16 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { static fromConfig( config: Config, - options: { logger: Logger; schedule: TaskRunner }, + options: { + logger: Logger; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; + }, ): GitlabDiscoveryEntityProvider[] { + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + const providerConfigs = readGitlabConfigs(config); const integrations = ScmIntegrations.fromConfig(config).gitlab; const providers: GitlabDiscoveryEntityProvider[] = []; @@ -64,11 +72,23 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { `No gitlab integration found that matches host ${providerConfig.host}`, ); } + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + providers.push( new GitlabDiscoveryEntityProvider({ ...options, config: providerConfig, integration, + taskRunner, }), ); }); @@ -79,14 +99,14 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { config: GitlabProviderConfig; integration: GitLabIntegration; logger: Logger; - schedule: TaskRunner; + taskRunner: TaskRunner; }) { this.config = options.config; this.integration = options.integration; this.logger = options.logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(options.schedule); + this.scheduleFn = this.createScheduleFn(options.taskRunner); } getProviderName(): string { @@ -98,10 +118,10 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { await this.scheduleFn(); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index cb36e5166c..6b337bf7cd 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readGitlabConfigs } from './config'; describe('config', () => { @@ -53,6 +54,7 @@ describe('config', () => { host: 'host', catalogFile: 'catalog-info.yaml', projectPattern: /[\s\S]*/, + schedule: undefined, }), ); }); @@ -83,6 +85,49 @@ describe('config', () => { host: 'host', catalogFile: 'custom-file.yaml', projectPattern: /[\s\S]*/, + schedule: undefined, + }), + ); + }); + + it('valid config with schedule', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + group: 'group', + host: 'host', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, + }, + }, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(1); + result.forEach(r => + expect(r).toStrictEqual({ + id: 'test', + group: 'group', + branch: 'master', + host: 'host', + catalogFile: 'catalog-info.yaml', + projectPattern: /[\s\S]*/, + schedule: { + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 3, + }, + initialDelay: undefined, + scope: undefined, + }, }), ); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index 392922860f..3dfc00745e 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GitlabProviderConfig } from '../lib'; @@ -35,6 +36,10 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { config.getOptionalString('projectPattern') ?? /[\s\S]*/, ); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, group, @@ -42,6 +47,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { host, catalogFile, projectPattern, + schedule, }; } diff --git a/yarn.lock b/yarn.lock index 787cb2a94a..4c1f1a9bb6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4622,6 +4622,7 @@ __metadata: "@types/lodash": ^4.14.151 "@types/uuid": ^8.0.0 lodash: ^4.17.21 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 uuid: ^8.0.0 From 6bb046bcbed233f5f63c4b82d2364e74b29a9bd1 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 12:29:42 +0200 Subject: [PATCH 11/11] feat(catalog/gitlab): Add backend plugin Add `gitlabDiscoveryEntityProviderCatalogModule` (new backend-plugin-api, alpha). Relates-to: PR #13859 Signed-off-by: Patrick Jungermann --- .changeset/chatty-planets-flash.md | 5 + .../api-report.md | 6 ++ .../package.json | 10 +- .../src/index.ts | 1 + ...scoveryEntityProviderCatalogModule.test.ts | 96 +++++++++++++++++++ ...labDiscoveryEntityProviderCatalogModule.ts | 53 ++++++++++ yarn.lock | 2 + 7 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 .changeset/chatty-planets-flash.md create mode 100644 plugins/catalog-backend-module-gitlab/src/service/GitlabDiscoveryEntityProviderCatalogModule.test.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/service/GitlabDiscoveryEntityProviderCatalogModule.ts diff --git a/.changeset/chatty-planets-flash.md b/.changeset/chatty-planets-flash.md new file mode 100644 index 0000000000..d6a5c6ad42 --- /dev/null +++ b/.changeset/chatty-planets-flash.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Add `gitlabDiscoveryEntityProviderCatalogModule` (new backend-plugin-api, alpha). diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index bd7b13efc3..9ed90a2c42 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; @@ -32,6 +33,11 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { refresh(logger: Logger): Promise; } +// @alpha +export const gitlabDiscoveryEntityProviderCatalogModule: ( + options?: undefined, +) => BackendFeature; + // @public export class GitLabDiscoveryProcessor implements CatalogProcessor { // (undocumented) diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index e537fa5e04..9946ac0997 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -7,6 +7,7 @@ "license": "Apache-2.0", "publishConfig": { "access": "public", + "alphaTypes": "dist/index.alpha.d.ts", "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, @@ -23,22 +24,24 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", + "start": "backstage-cli package start", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", - "clean": "backstage-cli package clean", - "start": "backstage-cli package start" + "clean": "backstage-cli package clean" }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", "@backstage/types": "workspace:^", "lodash": "^4.17.21", "msw": "^0.47.0", @@ -54,6 +57,7 @@ "luxon": "^3.0.0" }, "files": [ + "alpha", "config.d.ts", "dist" ], diff --git a/plugins/catalog-backend-module-gitlab/src/index.ts b/plugins/catalog-backend-module-gitlab/src/index.ts index 3182210c81..27aac9e78f 100644 --- a/plugins/catalog-backend-module-gitlab/src/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/index.ts @@ -22,3 +22,4 @@ export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; export { GitlabDiscoveryEntityProvider } from './providers'; +export { gitlabDiscoveryEntityProviderCatalogModule } from './service/GitlabDiscoveryEntityProviderCatalogModule'; diff --git a/plugins/catalog-backend-module-gitlab/src/service/GitlabDiscoveryEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-gitlab/src/service/GitlabDiscoveryEntityProviderCatalogModule.test.ts new file mode 100644 index 0000000000..0b803734b1 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/service/GitlabDiscoveryEntityProviderCatalogModule.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { + PluginTaskScheduler, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { startTestBackend } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { Duration } from 'luxon'; +import { gitlabDiscoveryEntityProviderCatalogModule } from './GitlabDiscoveryEntityProviderCatalogModule'; +import { GitlabDiscoveryEntityProvider } from '../providers'; + +describe('gitlabDiscoveryEntityProviderCatalogModule', () => { + it('should register provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = { + createScheduledTaskRunner: (schedule: TaskScheduleDefinition) => { + usedSchedule = schedule; + return runner; + }, + } as unknown as PluginTaskScheduler; + + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }); + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + services: [ + [configServiceRef, config], + [loggerServiceRef, getVoidLogger()], + [schedulerServiceRef, scheduler], + ], + features: [gitlabDiscoveryEntityProviderCatalogModule()], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + expect(runner).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab/src/service/GitlabDiscoveryEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-gitlab/src/service/GitlabDiscoveryEntityProviderCatalogModule.ts new file mode 100644 index 0000000000..dc11841e20 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/service/GitlabDiscoveryEntityProviderCatalogModule.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + createBackendModule, + loggerToWinstonLogger, + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { GitlabDiscoveryEntityProvider } from '../providers'; + +/** + * Registers the GitlabDiscoveryEntityProvider with the catalog processing extension point. + * + * @alpha + */ +export const gitlabDiscoveryEntityProviderCatalogModule = createBackendModule({ + pluginId: 'catalog', + moduleId: 'gitlabDiscoveryEntityProvider', + register(env) { + env.registerInit({ + deps: { + config: configServiceRef, + catalog: catalogProcessingExtensionPoint, + logger: loggerServiceRef, + scheduler: schedulerServiceRef, + }, + async init({ config, catalog, logger, scheduler }) { + catalog.addEntityProvider( + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger: loggerToWinstonLogger(logger), + scheduler, + }), + ); + }, + }); + }, +}); diff --git a/yarn.lock b/yarn.lock index 4c1f1a9bb6..cacca47910 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4610,6 +4610,7 @@ __metadata: resolution: "@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" @@ -4618,6 +4619,7 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 "@types/uuid": ^8.0.0