From 8d1a5e08ca5bbb7cb190619168f0643f844bd73e Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 15:23:48 +0200 Subject: [PATCH 1/2] feat(catalog/msgraph): Add option to configure schedule via `app-config.yaml` Relates-to: PR #13859 Relates-to: PR #14034 Signed-off-by: Patrick Jungermann --- .changeset/dirty-birds-burn.md | 8 + .../catalog-backend-module-msgraph/README.md | 26 +- .../api-report.md | 6 +- .../config.d.ts | 22 +- .../package.json | 1 + .../src/microsoftGraph/config.test.ts | 13 + .../src/microsoftGraph/config.ts | 14 + .../MicrosoftGraphOrgEntityProvider.test.ts | 265 ++++++++++++++---- .../MicrosoftGraphOrgEntityProvider.ts | 43 ++- yarn.lock | 1 + 10 files changed, 312 insertions(+), 87 deletions(-) create mode 100644 .changeset/dirty-birds-burn.md diff --git a/.changeset/dirty-birds-burn.md b/.changeset/dirty-birds-burn.md new file mode 100644 index 0000000000..98f2c5967d --- /dev/null +++ b/.changeset/dirty-birds-burn.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +`MicrosoftGraphOrgEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://github.com/backstage/backstage/tree/master/plugins/catalog-backend-module-msgraph#readme diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index 9bf979f6a6..6f8c95b667 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -85,6 +85,13 @@ catalog: # in order to add extra information to your groups that can be used on your custom groupTransformers # See https://docs.microsoft.com/en-us/graph/api/resources/schemaextension?view=graph-rest-1.0 select: ['id', 'displayName', 'description'] + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { hours: 1 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 50 } + # supports ISO duration, "human duration" as used in code + initialDelay: { seconds: 15}, ``` `user.filter` and `userGroupMember.filter` are mutually exclusive, only one can be provided. If both are provided, an error will be thrown. @@ -116,13 +123,21 @@ yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-msgraph + builder.addEntityProvider( + MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { + logger: env.logger, ++ scheduler, ++ }), ++ ); +``` + +Instead of configuring the refresh schedule inside the config (per provider instance), +you can define it in code (for all of them): + +```diff +- scheduler, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { hours: 1 }, + timeout: { minutes: 50 }, -+ initialDelay: { seconds: 15} ++ initialDelay: { seconds: 15}, + }), -+ }), -+ ); ``` ## Customize the Processor or Entity Provider @@ -161,10 +176,7 @@ export async function myGroupTransformer( builder.addEntityProvider( MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { logger: env.logger, - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 5 }, - timeout: { minutes: 3 }, - }), + scheduler, + groupTransformer: myGroupTransformer, }), ); diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 8b77ddf7c3..f5a77ceb51 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -12,8 +12,10 @@ import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Response as Response_2 } from 'node-fetch'; import { TaskRunner } from '@backstage/backend-tasks'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { TokenCredential } from '@azure/identity'; import { UserEntity } from '@backstage/catalog-model'; @@ -147,7 +149,8 @@ export type MicrosoftGraphOrgEntityProviderOptions = | MicrosoftGraphOrgEntityProviderLegacyOptions | { logger: Logger; - schedule: 'manual' | TaskRunner; + schedule?: 'manual' | TaskRunner; + scheduler?: PluginTaskScheduler; userTransformer?: UserTransformer | Record; groupTransformer?: GroupTransformer | Record; organizationTransformer?: @@ -202,6 +205,7 @@ export type MicrosoftGraphProviderConfig = { groupSearch?: string; groupSelect?: string[]; queryMode?: 'basic' | 'advanced'; + schedule?: TaskScheduleDefinition; }; // @public diff --git a/plugins/catalog-backend-module-msgraph/config.d.ts b/plugins/catalog-backend-module-msgraph/config.d.ts index 9fe1cb81c9..904ce1f646 100644 --- a/plugins/catalog-backend-module-msgraph/config.d.ts +++ b/plugins/catalog-backend-module-msgraph/config.d.ts @@ -14,14 +14,10 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { - /** - * Configuration options for the catalog plugin. - */ catalog?: { - /** - * List of processor-specific options and attributes - */ processors?: { /** * MicrosoftGraphOrgReaderProcessor configuration @@ -109,9 +105,7 @@ export interface Config { }>; }; }; - /** - * List of provider-specific options and attributes - */ + providers?: { /** * MicrosoftGraphOrgEntityProvider configuration. @@ -209,6 +203,11 @@ export interface Config { */ search?: string; }; + + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } | Record< string, @@ -292,6 +291,11 @@ export interface Config { */ search?: string; }; + + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-msgraph/package.json b/plugins/catalog-backend-module-msgraph/package.json index d3ac35f743..0a5aed29f9 100644 --- a/plugins/catalog-backend-module-msgraph/package.json +++ b/plugins/catalog-backend-module-msgraph/package.json @@ -51,6 +51,7 @@ "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", + "luxon": "^3.0.0", "msw": "^0.47.0" }, "files": [ diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index 3331e3609d..68c174665d 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readMicrosoftGraphConfig, readProviderConfigs } from './config'; describe('readMicrosoftGraphConfig', () => { @@ -172,6 +173,12 @@ describe('readProviderConfigs', () => { filter: 'securityEnabled eq false', select: ['id', 'displayName', 'description'], }, + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, }, }, }, @@ -192,6 +199,12 @@ describe('readProviderConfigs', () => { groupExpand: 'member', groupSelect: ['id', 'displayName', 'description'], groupFilter: 'securityEnabled eq false', + schedule: { + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 3, + }, + }, }, ]; expect(actual).toEqual(expected); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index b8565e0d72..506951ff10 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -14,6 +14,10 @@ * limitations under the License. */ +import { + readTaskScheduleDefinitionFromConfig, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { trimEnd } from 'lodash'; @@ -121,6 +125,11 @@ export type MicrosoftGraphProviderConfig = { * Some features like `$expand` are not available for advanced queries, though. */ queryMode?: 'basic' | 'advanced'; + + /** + * Schedule configuration for refresh tasks. + */ + schedule?: TaskScheduleDefinition; }; /** @@ -296,6 +305,10 @@ export function readProviderConfig( throw new Error(`clientId must be provided when clientSecret is defined.`); } + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, target, @@ -312,5 +325,6 @@ export function readProviderConfig( queryMode, userGroupMemberFilter, userGroupMemberSearch, + schedule, }; } diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts index ff68235c4c..8af327cf27 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts @@ -14,6 +14,11 @@ * limitations under the License. */ import { getVoidLogger } from '@backstage/backend-common'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { ANNOTATION_LOCATION, @@ -43,10 +48,21 @@ const readMicrosoftGraphOrgMocked = readMicrosoftGraphOrg as jest.Mock< Promise<{ users: UserEntity[]; groups: GroupEntity[] }> >; -describe('MicrosoftGraphOrgEntityProvider', () => { - afterEach(() => jest.resetAllMocks()); +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; - it('should apply mutation', async () => { + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +describe('MicrosoftGraphOrgEntityProvider', () => { + beforeEach(() => { jest .spyOn(MicrosoftGraphClient, 'create') .mockReturnValue({} as unknown as MicrosoftGraphClient); @@ -78,8 +94,65 @@ describe('MicrosoftGraphOrgEntityProvider', () => { }, ], }); + }); - const config = { + afterEach(() => jest.resetAllMocks()); + + const logger = getVoidLogger(); + const taskRunner = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => taskRunner, + } as unknown as PluginTaskScheduler; + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const expectedMutation = { + entities: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'msgraph:customProviderId/u1', + 'backstage.io/managed-by-origin-location': + 'msgraph:customProviderId/u1', + }, + name: 'u1', + }, + spec: { + memberOf: [], + }, + }, + locationKey: 'msgraph-org-provider:customProviderId', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'msgraph:customProviderId/g1', + 'backstage.io/managed-by-origin-location': + 'msgraph:customProviderId/g1', + }, + name: 'g1', + }, + spec: { + children: [], + type: 'team', + }, + }, + locationKey: 'msgraph-org-provider:customProviderId', + }, + ], + type: 'full', + }; + + it('should apply mutation - manual', async () => { + const config = new ConfigReader({ catalog: { providers: { microsoftGraphOrg: { @@ -92,67 +165,143 @@ describe('MicrosoftGraphOrgEntityProvider', () => { }, }, }, - }; - const entityProviderConnection: EntityProviderConnection = { - applyMutation: jest.fn(), - refresh: jest.fn(), - }; - const provider = MicrosoftGraphOrgEntityProvider.fromConfig( - new ConfigReader(config), - { - logger: getVoidLogger(), - schedule: 'manual', - }, - )[0]; - - provider.connect(entityProviderConnection); + }); + const provider = MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + schedule: 'manual', + })[0]; + await provider.connect(entityProviderConnection); await provider.read(); - expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'msgraph:customProviderId/u1', - 'backstage.io/managed-by-origin-location': - 'msgraph:customProviderId/u1', - }, - name: 'u1', - }, - spec: { - memberOf: [], + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith( + expectedMutation, + ); + }); + + it('should apply mutation - schedule', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', }, }, - locationKey: 'msgraph-org-provider:customProviderId', }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'msgraph:customProviderId/g1', - 'backstage.io/managed-by-origin-location': - 'msgraph:customProviderId/g1', - }, - name: 'g1', - }, - spec: { - children: [], - type: 'team', - }, - }, - locationKey: 'msgraph-org-provider:customProviderId', - }, - ], - type: 'full', + }, }); + const provider = MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + schedule: taskRunner, + })[0]; + expect(provider.getProviderName()).toEqual( + 'MicrosoftGraphOrgEntityProvider:customProviderId', + ); + + await provider.connect(entityProviderConnection); + + const taskDef = taskRunner.getTasks()[0]; + expect(taskDef.id).toEqual( + 'MicrosoftGraphOrgEntityProvider:customProviderId:refresh', + ); + await (taskDef.fn as () => Promise)(); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith( + expectedMutation, + ); + }); + + it('should apply mutation - scheduler', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }); + const provider = MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + scheduler, + })[0]; + expect(provider.getProviderName()).toEqual( + 'MicrosoftGraphOrgEntityProvider:customProviderId', + ); + + await provider.connect(entityProviderConnection); + + const taskDef = taskRunner.getTasks()[0]; + expect(taskDef.id).toEqual( + 'MicrosoftGraphOrgEntityProvider:customProviderId:refresh', + ); + await (taskDef.fn as () => Promise)(); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith( + expectedMutation, + ); + }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + }, + }, + }, + }, + }); + + expect(() => + MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + }, + }, + }, + }, + }); + + expect(() => + MicrosoftGraphOrgEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for MicrosoftGraphOrgEntityProvider:customProviderId', + ); }); }); diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 67f3039873..32892e7201 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -67,7 +67,13 @@ export type MicrosoftGraphOrgEntityProviderOptions = * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} * to enable automatic scheduling of tasks. */ - schedule: 'manual' | TaskRunner; + schedule?: 'manual' | TaskRunner; + + /** + * Scheduler used to schedule refreshes based on + * the schedule config. + */ + scheduler?: PluginTaskScheduler; /** * The function that transforms a user entry in msgraph to an entity. @@ -168,6 +174,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { ]; } + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + function getTransformer( id: string, transformers?: T | Record, @@ -180,6 +190,16 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { } return readProviderConfigs(configRoot).map(providerConfig => { + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for MicrosoftGraphOrgEntityProvider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + const provider = new MicrosoftGraphOrgEntityProvider({ id: providerConfig.id, provider: providerConfig, @@ -197,7 +217,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { options.organizationTransformer, ), }); - provider.schedule(options.schedule); + + if (taskRunner !== 'manual') { + provider.schedule(taskRunner); + } return provider; }); @@ -238,7 +261,9 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { provider, }); - result.schedule(options.schedule); + if (options.schedule !== 'manual') { + result.schedule(options.schedule); + } return result; } @@ -311,16 +336,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { markCommitComplete(); } - private schedule( - schedule: MicrosoftGraphOrgEntityProviderOptions['schedule'], - ) { - if (schedule === 'manual') { - return; - } - + private schedule(taskRunner: TaskRunner) { this.scheduleFn = async () => { const id = `${this.getProviderName()}:refresh`; - await schedule.run({ + await taskRunner.run({ id, fn: async () => { const logger = this.options.logger.child({ diff --git a/yarn.lock b/yarn.lock index ac27eb8215..0c9a4343ba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4652,6 +4652,7 @@ __metadata: "@types/lodash": ^4.14.151 "@types/node-fetch": ^2.5.12 lodash: ^4.17.21 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 p-limit: ^3.0.2 From 384f99c2766868b2c067e594afe16a65dc4fbe95 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 7 Oct 2022 15:52:22 +0200 Subject: [PATCH 2/2] feat(catalog/msgraph): Add backend plugin Add `microsoftGraphOrgEntityProviderCatalogModule` (new backend-plugin-api, alpha). Relates-to: PR #13859 Relates-to: PR #14034 Signed-off-by: Patrick Jungermann --- .changeset/mean-files-fly.md | 5 + .../api-report.md | 15 +++ .../package.json | 14 ++- .../src/index.ts | 4 +- ...raphOrgEntityProviderCatalogModule.test.ts | 89 ++++++++++++++++++ ...softGraphOrgEntityProviderCatalogModule.ts | 93 +++++++++++++++++++ yarn.lock | 2 + 7 files changed, 216 insertions(+), 6 deletions(-) create mode 100644 .changeset/mean-files-fly.md create mode 100644 plugins/catalog-backend-module-msgraph/src/service/MicrosoftGraphOrgEntityProviderCatalogModule.test.ts create mode 100644 plugins/catalog-backend-module-msgraph/src/service/MicrosoftGraphOrgEntityProviderCatalogModule.ts diff --git a/.changeset/mean-files-fly.md b/.changeset/mean-files-fly.md new file mode 100644 index 0000000000..a7a79bcf74 --- /dev/null +++ b/.changeset/mean-files-fly.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Add `microsoftGraphOrgEntityProviderCatalogModule` (new backend-plugin-api, alpha). diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index f5a77ceb51..921d682b89 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; @@ -133,6 +134,20 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { read(options?: { logger?: Logger }): Promise; } +// @alpha +export const microsoftGraphOrgEntityProviderCatalogModule: ( + options?: MicrosoftGraphOrgEntityProviderCatalogModuleOptions | undefined, +) => BackendFeature; + +// @alpha +export interface MicrosoftGraphOrgEntityProviderCatalogModuleOptions { + groupTransformer?: GroupTransformer | Record; + organizationTransformer?: + | OrganizationTransformer + | Record; + userTransformer?: UserTransformer | Record; +} + // @public @deprecated export interface MicrosoftGraphOrgEntityProviderLegacyOptions { groupTransformer?: GroupTransformer; diff --git a/plugins/catalog-backend-module-msgraph/package.json b/plugins/catalog-backend-module-msgraph/package.json index 0a5aed29f9..91a3b9973a 100644 --- a/plugins/catalog-backend-module-msgraph/package.json +++ b/plugins/catalog-backend-module-msgraph/package.json @@ -7,6 +7,7 @@ "license": "Apache-2.0", "publishConfig": { "access": "public", + "alphaTypes": "dist/index.alpha.d.ts", "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, @@ -23,20 +24,22 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", + "start": "backstage-cli package start", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", - "clean": "backstage-cli package clean", - "start": "backstage-cli package start" + "clean": "backstage-cli package clean" }, "dependencies": { "@azure/identity": "^2.1.0", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", "@microsoft/microsoft-graph-types": "^2.6.0", "@types/node-fetch": "^2.5.12", "lodash": "^4.17.21", @@ -55,8 +58,9 @@ "msw": "^0.47.0" }, "files": [ - "dist", - "config.d.ts" + "alpha", + "config.d.ts", + "dist" ], "configSchema": "config.d.ts" } diff --git a/plugins/catalog-backend-module-msgraph/src/index.ts b/plugins/catalog-backend-module-msgraph/src/index.ts index 6a7adedc58..b80a0b3863 100644 --- a/plugins/catalog-backend-module-msgraph/src/index.ts +++ b/plugins/catalog-backend-module-msgraph/src/index.ts @@ -20,5 +20,7 @@ * @packageDocumentation */ -export * from './processors'; export * from './microsoftGraph'; +export * from './processors'; +export { microsoftGraphOrgEntityProviderCatalogModule } from './service/MicrosoftGraphOrgEntityProviderCatalogModule'; +export type { MicrosoftGraphOrgEntityProviderCatalogModuleOptions } from './service/MicrosoftGraphOrgEntityProviderCatalogModule'; diff --git a/plugins/catalog-backend-module-msgraph/src/service/MicrosoftGraphOrgEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-msgraph/src/service/MicrosoftGraphOrgEntityProviderCatalogModule.test.ts new file mode 100644 index 0000000000..764f504898 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/service/MicrosoftGraphOrgEntityProviderCatalogModule.test.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { + configServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { + PluginTaskScheduler, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { startTestBackend } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { Duration } from 'luxon'; +import { microsoftGraphOrgEntityProviderCatalogModule } from './MicrosoftGraphOrgEntityProviderCatalogModule'; +import { MicrosoftGraphOrgEntityProvider } from '../processors'; + +describe('awsS3EntityProviderCatalogModule', () => { + it('should register provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = { + createScheduledTaskRunner: (schedule: TaskScheduleDefinition) => { + usedSchedule = schedule; + return runner; + }, + } as unknown as PluginTaskScheduler; + + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }); + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + services: [ + [configServiceRef, config], + [loggerServiceRef, getVoidLogger()], + [schedulerServiceRef, scheduler], + ], + features: [microsoftGraphOrgEntityProviderCatalogModule()], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('PT30M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'MicrosoftGraphOrgEntityProvider:customProviderId', + ); + expect(runner).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/catalog-backend-module-msgraph/src/service/MicrosoftGraphOrgEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-msgraph/src/service/MicrosoftGraphOrgEntityProviderCatalogModule.ts new file mode 100644 index 0000000000..9e817636fa --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/service/MicrosoftGraphOrgEntityProviderCatalogModule.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + configServiceRef, + createBackendModule, + loggerServiceRef, + loggerToWinstonLogger, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { + GroupTransformer, + OrganizationTransformer, + UserTransformer, +} from '../microsoftGraph'; +import { MicrosoftGraphOrgEntityProvider } from '../processors'; + +/** + * Options for {@link microsoftGraphOrgEntityProviderCatalogModule}. + * + * @alpha + */ +export interface MicrosoftGraphOrgEntityProviderCatalogModuleOptions { + /** + * The function that transforms a user entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + userTransformer?: UserTransformer | Record; + + /** + * The function that transforms a group entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + groupTransformer?: GroupTransformer | Record; + + /** + * The function that transforms an organization entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + organizationTransformer?: + | OrganizationTransformer + | Record; +} + +/** + * Registers the MicrosoftGraphOrgEntityProvider with the catalog processing extension point. + * + * @alpha + */ +export const microsoftGraphOrgEntityProviderCatalogModule = createBackendModule( + { + pluginId: 'catalog', + moduleId: 'microsoftGraphOrgEntityProvider', + register( + env, + options?: MicrosoftGraphOrgEntityProviderCatalogModuleOptions, + ) { + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + config: configServiceRef, + logger: loggerServiceRef, + scheduler: schedulerServiceRef, + }, + async init({ catalog, config, logger, scheduler }) { + catalog.addEntityProvider( + MicrosoftGraphOrgEntityProvider.fromConfig(config, { + groupTransformer: options?.groupTransformer, + logger: loggerToWinstonLogger(logger), + organizationTransformer: options?.organizationTransformer, + scheduler, + userTransformer: options?.userTransformer, + }), + ); + }, + }); + }, + }, +); diff --git a/yarn.lock b/yarn.lock index 0c9a4343ba..fbc337dac6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4642,12 +4642,14 @@ __metadata: dependencies: "@azure/identity": ^2.1.0 "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" "@microsoft/microsoft-graph-types": ^2.6.0 "@types/lodash": ^4.14.151 "@types/node-fetch": ^2.5.12