diff --git a/.changeset/dirty-birds-burn.md b/.changeset/dirty-birds-burn.md new file mode 100644 index 0000000000..98f2c5967d --- /dev/null +++ b/.changeset/dirty-birds-burn.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +`MicrosoftGraphOrgEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://github.com/backstage/backstage/tree/master/plugins/catalog-backend-module-msgraph#readme diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index 9bf979f6a6..6f8c95b667 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -85,6 +85,13 @@ catalog: # in order to add extra information to your groups that can be used on your custom groupTransformers # See https://docs.microsoft.com/en-us/graph/api/resources/schemaextension?view=graph-rest-1.0 select: ['id', 'displayName', 'description'] + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { hours: 1 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 50 } + # supports ISO duration, "human duration" as used in code + initialDelay: { seconds: 15}, ``` `user.filter` and `userGroupMember.filter` are mutually exclusive, only one can be provided. If both are provided, an error will be thrown. @@ -116,13 +123,21 @@ yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-msgraph + builder.addEntityProvider( + MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { + logger: env.logger, ++ scheduler, ++ }), ++ ); +``` + +Instead of configuring the refresh schedule inside the config (per provider instance), +you can define it in code (for all of them): + +```diff +- scheduler, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { hours: 1 }, + timeout: { minutes: 50 }, -+ initialDelay: { seconds: 15} ++ initialDelay: { seconds: 15}, + }), -+ }), -+ ); ``` ## Customize the Processor or Entity Provider @@ -161,10 +176,7 @@ export async function myGroupTransformer( builder.addEntityProvider( MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { logger: env.logger, - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 5 }, - timeout: { minutes: 3 }, - }), + scheduler, + groupTransformer: myGroupTransformer, }), ); diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 8b77ddf7c3..f5a77ceb51 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -12,8 +12,10 @@ import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Response as Response_2 } from 'node-fetch'; import { TaskRunner } from '@backstage/backend-tasks'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { TokenCredential } from '@azure/identity'; import { UserEntity } from '@backstage/catalog-model'; @@ -147,7 +149,8 @@ export type MicrosoftGraphOrgEntityProviderOptions = | MicrosoftGraphOrgEntityProviderLegacyOptions | { logger: Logger; - schedule: 'manual' | TaskRunner; + schedule?: 'manual' | TaskRunner; + scheduler?: PluginTaskScheduler; userTransformer?: UserTransformer | Record; groupTransformer?: GroupTransformer | Record; organizationTransformer?: @@ -202,6 +205,7 @@ export type MicrosoftGraphProviderConfig = { groupSearch?: string; groupSelect?: string[]; queryMode?: 'basic' | 'advanced'; + schedule?: TaskScheduleDefinition; }; // @public diff --git a/plugins/catalog-backend-module-msgraph/config.d.ts b/plugins/catalog-backend-module-msgraph/config.d.ts index 9fe1cb81c9..904ce1f646 100644 --- a/plugins/catalog-backend-module-msgraph/config.d.ts +++ b/plugins/catalog-backend-module-msgraph/config.d.ts @@ -14,14 +14,10 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { - /** - * Configuration options for the catalog plugin. - */ catalog?: { - /** - * List of processor-specific options and attributes - */ processors?: { /** * MicrosoftGraphOrgReaderProcessor configuration @@ -109,9 +105,7 @@ export interface Config { }>; }; }; - /** - * List of provider-specific options and attributes - */ + providers?: { /** * MicrosoftGraphOrgEntityProvider configuration. @@ -209,6 +203,11 @@ export interface Config { */ search?: string; }; + + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } | Record< string, @@ -292,6 +291,11 @@ export interface Config { */ search?: string; }; + + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-msgraph/package.json b/plugins/catalog-backend-module-msgraph/package.json index d3ac35f743..0a5aed29f9 100644 --- a/plugins/catalog-backend-module-msgraph/package.json +++ b/plugins/catalog-backend-module-msgraph/package.json @@ -51,6 +51,7 @@ "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", + "luxon": "^3.0.0", "msw": "^0.47.0" }, "files": [ diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index 3331e3609d..68c174665d 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readMicrosoftGraphConfig, readProviderConfigs } from './config'; describe('readMicrosoftGraphConfig', () => { @@ -172,6 +173,12 @@ describe('readProviderConfigs', () => { filter: 'securityEnabled eq false', select: ['id', 'displayName', 'description'], }, + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, }, }, }, @@ -192,6 +199,12 @@ describe('readProviderConfigs', () => { groupExpand: 'member', groupSelect: ['id', 'displayName', 'description'], groupFilter: 'securityEnabled eq false', + schedule: { + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 3, + }, + }, }, ]; expect(actual).toEqual(expected); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index b8565e0d72..506951ff10 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -14,6 +14,10 @@ * limitations under the License. */ +import { + readTaskScheduleDefinitionFromConfig, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { trimEnd } from 'lodash'; @@ -121,6 +125,11 @@ export type MicrosoftGraphProviderConfig = { * Some features like `$expand` are not available for advanced queries, though. */ queryMode?: 'basic' | 'advanced'; + + /** + * Schedule configuration for refresh tasks. + */ + schedule?: TaskScheduleDefinition; }; /** @@ -296,6 +305,10 @@ export function readProviderConfig( throw new Error(`clientId must be provided when clientSecret is defined.`); } + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, target, @@ -312,5 +325,6 @@ export function readProviderConfig( queryMode, userGroupMemberFilter, userGroupMemberSearch, + schedule, }; } diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts index ff68235c4c..8af327cf27 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts @@ -14,6 +14,11 @@ * limitations under the License. */ import { getVoidLogger } from '@backstage/backend-common'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { ANNOTATION_LOCATION, @@ -43,10 +48,21 @@ const readMicrosoftGraphOrgMocked = readMicrosoftGraphOrg as jest.Mock< Promise<{ users: UserEntity[]; groups: GroupEntity[] }> >; -describe('MicrosoftGraphOrgEntityProvider', () => { - afterEach(() => jest.resetAllMocks()); +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; - it('should apply mutation', async () => { + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +describe('MicrosoftGraphOrgEntityProvider', () => { + beforeEach(() => { jest .spyOn(MicrosoftGraphClient, 'create') .mockReturnValue({} as unknown as MicrosoftGraphClient); @@ -78,8 +94,65 @@ describe('MicrosoftGraphOrgEntityProvider', () => { }, ], }); + }); - const config = { + afterEach(() => jest.resetAllMocks()); + + const logger = getVoidLogger(); + const taskRunner = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => taskRunner, + } as unknown as PluginTaskScheduler; + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const expectedMutation = { + entities: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'msgraph:customProviderId/u1', + 'backstage.io/managed-by-origin-location': + 'msgraph:customProviderId/u1', + }, + name: 'u1', + }, + spec: { + memberOf: [], + }, + }, + locationKey: 'msgraph-org-provider:customProviderId', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'msgraph:customProviderId/g1', + 'backstage.io/managed-by-origin-location': + 'msgraph:customProviderId/g1', + }, + name: 'g1', + }, + spec: { + children: [], + type: 'team', + }, + }, + locationKey: 'msgraph-org-provider:customProviderId', + }, + ], + type: 'full', + }; + + it('should apply mutation - manual', async () => { + const config = new ConfigReader({ catalog: { providers: { microsoftGraphOrg: { @@ -92,67 +165,143 @@ describe('MicrosoftGraphOrgEntityProvider', () => { }, }, }, - }; - const entityProviderConnection: EntityProviderConnection = { - applyMutation: jest.fn(), - refresh: jest.fn(), - }; - const provider = MicrosoftGraphOrgEntityProvider.fromConfig( - new ConfigReader(config), - { - logger: getVoidLogger(), - schedule: 'manual', - }, - )[0]; - - provider.connect(entityProviderConnection); + }); + const provider = MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + schedule: 'manual', + })[0]; + await provider.connect(entityProviderConnection); await provider.read(); - expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'msgraph:customProviderId/u1', - 'backstage.io/managed-by-origin-location': - 'msgraph:customProviderId/u1', - }, - name: 'u1', - }, - spec: { - memberOf: [], + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith( + expectedMutation, + ); + }); + + it('should apply mutation - schedule', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', }, }, - locationKey: 'msgraph-org-provider:customProviderId', }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'msgraph:customProviderId/g1', - 'backstage.io/managed-by-origin-location': - 'msgraph:customProviderId/g1', - }, - name: 'g1', - }, - spec: { - children: [], - type: 'team', - }, - }, - locationKey: 'msgraph-org-provider:customProviderId', - }, - ], - type: 'full', + }, }); + const provider = MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + schedule: taskRunner, + })[0]; + expect(provider.getProviderName()).toEqual( + 'MicrosoftGraphOrgEntityProvider:customProviderId', + ); + + await provider.connect(entityProviderConnection); + + const taskDef = taskRunner.getTasks()[0]; + expect(taskDef.id).toEqual( + 'MicrosoftGraphOrgEntityProvider:customProviderId:refresh', + ); + await (taskDef.fn as () => Promise)(); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith( + expectedMutation, + ); + }); + + it('should apply mutation - scheduler', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }); + const provider = MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + scheduler, + })[0]; + expect(provider.getProviderName()).toEqual( + 'MicrosoftGraphOrgEntityProvider:customProviderId', + ); + + await provider.connect(entityProviderConnection); + + const taskDef = taskRunner.getTasks()[0]; + expect(taskDef.id).toEqual( + 'MicrosoftGraphOrgEntityProvider:customProviderId:refresh', + ); + await (taskDef.fn as () => Promise)(); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith( + expectedMutation, + ); + }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + }, + }, + }, + }, + }); + + expect(() => + MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + }, + }, + }, + }, + }); + + expect(() => + MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for MicrosoftGraphOrgEntityProvider:customProviderId', + ); }); }); diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 67f3039873..32892e7201 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -67,7 +67,13 @@ export type MicrosoftGraphOrgEntityProviderOptions = * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} * to enable automatic scheduling of tasks. */ - schedule: 'manual' | TaskRunner; + schedule?: 'manual' | TaskRunner; + + /** + * Scheduler used to schedule refreshes based on + * the schedule config. + */ + scheduler?: PluginTaskScheduler; /** * The function that transforms a user entry in msgraph to an entity. @@ -168,6 +174,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { ]; } + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + function getTransformer( id: string, transformers?: T | Record, @@ -180,6 +190,16 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { } return readProviderConfigs(configRoot).map(providerConfig => { + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for MicrosoftGraphOrgEntityProvider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + const provider = new MicrosoftGraphOrgEntityProvider({ id: providerConfig.id, provider: providerConfig, @@ -197,7 +217,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { options.organizationTransformer, ), }); - provider.schedule(options.schedule); + + if (taskRunner !== 'manual') { + provider.schedule(taskRunner); + } return provider; }); @@ -238,7 +261,9 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { provider, }); - result.schedule(options.schedule); + if (options.schedule !== 'manual') { + result.schedule(options.schedule); + } return result; } @@ -311,16 +336,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { markCommitComplete(); } - private schedule( - schedule: MicrosoftGraphOrgEntityProviderOptions['schedule'], - ) { - if (schedule === 'manual') { - return; - } - + private schedule(taskRunner: TaskRunner) { this.scheduleFn = async () => { const id = `${this.getProviderName()}:refresh`; - await schedule.run({ + await taskRunner.run({ id, fn: async () => { const logger = this.options.logger.child({ diff --git a/yarn.lock b/yarn.lock index ac27eb8215..0c9a4343ba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4652,6 +4652,7 @@ __metadata: "@types/lodash": ^4.14.151 "@types/node-fetch": ^2.5.12 lodash: ^4.17.21 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 p-limit: ^3.0.2