From a145672f0f273ffbce6db6d1afbc2cf14ca6627b Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Wed, 22 Jun 2022 00:18:59 +0200 Subject: [PATCH] feat: align msgraph provider config Align msgraph entity provider config with the config of other entity providers. Closes: #12065 Signed-off-by: Patrick Jungermann --- .changeset/long-bananas-rescue.md | 91 ++++++++++ app-config.yaml | 10 -- .../catalog-backend-module-msgraph/README.md | 144 ++++++--------- .../api-report.md | 22 ++- .../config.d.ts | 169 ++++++++++++++++++ .../src/microsoftGraph/config.test.ts | 133 +++++++++++++- .../src/microsoftGraph/config.ts | 100 ++++++++++- .../MicrosoftGraphOrgEntityProvider.test.ts | 46 +++-- .../MicrosoftGraphOrgEntityProvider.ts | 108 ++++++++++- .../MicrosoftGraphOrgReaderProcessor.test.ts | 1 + .../MicrosoftGraphOrgReaderProcessor.ts | 6 +- .../src/processors/index.ts | 5 +- 12 files changed, 710 insertions(+), 125 deletions(-) create mode 100644 .changeset/long-bananas-rescue.md diff --git a/.changeset/long-bananas-rescue.md b/.changeset/long-bananas-rescue.md new file mode 100644 index 0000000000..d990fd67c6 --- /dev/null +++ b/.changeset/long-bananas-rescue.md @@ -0,0 +1,91 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': minor +--- + +Align `msgraph` plugin's entity provider config with other providers. **Deprecated** entity processor as well as previous config. + +You will see warning at the log output until you migrate to the new setup. +All deprecated parts will be removed eventually after giving some time to migrate. + +Please find information on how to migrate your current setup to the new one below. + +**Migration Guide:** + +There were two different way on how to use the msgraph plugin: processor or provider. + +Previous registration for the processor: + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addProcessor( + MicrosoftGraphOrgReaderProcessor.fromConfig(env.config, { + logger: env.logger, + // [...] + }), +); +``` + +Previous registration when using the provider: + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addEntityProvider( + MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { + id: 'https://graph.microsoft.com/v1.0', + target: 'https://graph.microsoft.com/v1.0', + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // [...] + }), +); +``` + +Previous configuration as used for both: + +```yaml +# app-config.yaml +catalog: + processors: + microsoftGraphOrg: + providers: + - target: https://graph.microsoft.com/v1.0 + # [...] +``` + +**Replacement:** + +Please check https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-msgraph/README.md for the complete documentation of all configuration options (config as well as registration of the provider). + +```yaml +# app-config.yaml +catalog: + providers: + microsoftGraphOrg: + # In case you used the deprecated configuration with the entity provider + # using the value of `target` will keep the same location key for all + providerId: # some stable ID which will be used as part of the location key for all ingested data + target: https://graph.microsoft.com/v1.0 + # [...] +``` + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addEntityProvider( + MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // [...] + }), +); +``` + +In case you've used multiple entity providers before +**and** you had different transformers for each of them +you can provide these directly at the one `fromConfig` call +by passing a Record with the provider ID as key. diff --git a/app-config.yaml b/app-config.yaml index a56a3df9d1..9a3206026e 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -234,16 +234,6 @@ catalog: # dn: ou=access,ou=groups,ou=example,dc=example,dc=net # options: # filter: (&(objectClass=some-group-class)(!(groupType=email))) - microsoftGraphOrg: - ### Example for how to add your Microsoft Graph tenant - #providers: - # - target: https://graph.microsoft.com/v1.0 - # authority: https://login.microsoftonline.com - # tenantId: ${MICROSOFT_GRAPH_TENANT_ID} - # clientId: ${MICROSOFT_GRAPH_CLIENT_ID} - # clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} - # userFilter: accountEnabled eq true and userType eq 'member' - # groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') locations: # Add a location here to ingest it, for example from a URL: diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index e6429152e4..6132d5abc6 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -1,85 +1,86 @@ # Catalog Backend Module for Microsoft Graph -This is an extension module to the `plugin-catalog-backend` plugin, providing a -`MicrosoftGraphOrgReaderProcessor` and a `MicrosoftGraphOrgEntityProvider` that -can be used to ingest organization data from the Microsoft Graph API. This -processor is useful if you want to import users and groups from Azure Active -Directory or Office 365. +This is an extension module to the `plugin-catalog-backend` plugin, providing a `MicrosoftGraphOrgEntityProvider` +that can be used to ingest organization data from the Microsoft Graph API. +This provider is useful if you want to import users and groups from Azure Active Directory or Office 365. ## Getting Started -First you need to decide whether you want to use an [entity provider or a processor](https://backstage.io/docs/features/software-catalog/life-of-an-entity#stitching) to ingest the organization data. -If you want groups and users deleted from the source to be automatically deleted -from Backstage, choose the entity provider. - 1. Create or use an existing App registration in the [Microsoft Azure Portal](https://portal.azure.com/). The App registration requires at least the API permissions `Group.Read.All`, `GroupMember.Read.All`, `User.Read` and `User.Read.All` for Microsoft Graph (if you still run into errors about insufficient privileges, add `Team.ReadBasic.All` and `TeamMember.Read.All` too). -2. Configure the processor or entity provider: +2. Configure the entity provider: ```yaml # app-config.yaml catalog: - processors: + providers: microsoftGraphOrg: - providers: - - target: https://graph.microsoft.com/v1.0 - authority: https://login.microsoftonline.com - # If you don't know you tenantId, you can use Microsoft Graph Explorer - # to query it - tenantId: ${MICROSOFT_GRAPH_TENANT_ID} - # Client Id and Secret can be created under Certificates & secrets in - # the App registration in the Microsoft Azure Portal. - clientId: ${MICROSOFT_GRAPH_CLIENT_ID} - clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} - # Optional mode for querying which defaults to "basic". - # By default, the Microsoft Graph API only provides the basic feature set - # for querying. Certain features are limited to advanced querying capabilities. - # (See https://docs.microsoft.com/en-us/graph/aad-advanced-queries) - queryMode: basic # basic | advanced + providerId: + target: https://graph.microsoft.com/v1.0 + authority: https://login.microsoftonline.com + # If you don't know you tenantId, you can use Microsoft Graph Explorer + # to query it + tenantId: ${MICROSOFT_GRAPH_TENANT_ID} + # Client Id and Secret can be created under Certificates & secrets in + # the App registration in the Microsoft Azure Portal. + clientId: ${MICROSOFT_GRAPH_CLIENT_ID} + clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} + # Optional mode for querying which defaults to "basic". + # By default, the Microsoft Graph API only provides the basic feature set + # for querying. Certain features are limited to advanced querying capabilities. + # (See https://docs.microsoft.com/en-us/graph/aad-advanced-queries) + queryMode: basic # basic | advanced + # Optional configuration block + user: # Optional parameter to include the expanded resource or collection referenced # by a single relationship (navigation property) in your results. # Only one relationship can be expanded in a single request. # See https://docs.microsoft.com/en-us/graph/query-parameters#expand-parameter # Can be combined with userGroupMember[...] instead of userFilter. - userExpand: manager + expand: manager # Optional filter for user, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/api/resources/user?view=graph-rest-1.0#properties # and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter # This and userGroupMemberFilter are mutually exclusive, only one can be specified - userFilter: accountEnabled eq true and userType eq 'member' + filter: accountEnabled eq true and userType eq 'member' + # Optional configuration block + userGroupMember: # Optional filter for users, use group membership to get users. # (Filtered groups and fetch their members.) # This and userFilter are mutually exclusive, only one can be specified # See https://docs.microsoft.com/en-us/graph/search-query-parameter - userGroupMemberFilter: "displayName eq 'Backstage Users'" + filter: "displayName eq 'Backstage Users'" + # Optional search for users, use group membership to get users. + # (Search for groups and fetch their members.) + # This and userFilter are mutually exclusive, only one can be specified + search: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' + # Optional configuration block + group: # Optional parameter to include the expanded resource or collection referenced # by a single relationship (navigation property) in your results. # Only one relationship can be expanded in a single request. # See https://docs.microsoft.com/en-us/graph/query-parameters#expand-parameter # Can be combined with userGroupMember[...] instead of userFilter. - groupExpand: member - # Optional search for users, use group membership to get users. - # (Search for groups and fetch their members.) - # This and userFilter are mutually exclusive, only one can be specified - userGroupMemberSearch: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' + expand: member # Optional filter for group, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties - groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') + filter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') # Optional search for groups, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/search-query-parameter - groupSearch: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' - # Optional select for groups, this will allow you work with schemaExtensions in order to add extra information to your groups that can be used on you custom groupTransformers + search: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' + # Optional select for groups, this will allow you work with schemaExtensions + # in order to add extra information to your groups that can be used on you custom groupTransformers # See https://docs.microsoft.com/en-us/graph/api/resources/schemaextension?view=graph-rest-1.0 - groupSelect: ['id', 'displayName', 'description'] + select: ['id', 'displayName', 'description'] ``` -`userFilter` and `userGroupMemberFilter` are mutually exclusive, only one can be provided. If both are provided, an error will be thrown. +`user.filter` and `userGroupMember.filter` are mutually exclusive, only one can be provided. If both are provided, an error will be thrown. -By default, all users are loaded. If you want to filter users based on their attributes, use `userFilter`. `userGroupMemberFilter` can be used if you want to load users based on their group membership. +By default, all users are loaded. If you want to filter users based on their attributes, use `user.filter`. `userGroupMember.filter` can be used if you want to load users based on their group membership. 3. The package is not installed by default, therefore you have to add a dependency to `@backstage/plugin-catalog-backend-module-msgraph` to your @@ -90,15 +91,12 @@ By default, all users are loaded. If you want to filter users based on their att yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-msgraph ``` -### Using the Entity Provider - 4. The `MicrosoftGraphOrgEntityProvider` is not registered by default, so you have to register it in the catalog plugin. Pass the target to reference a provider from the configuration. ```diff // packages/backend/src/plugins/catalog.ts -+import { Duration } from 'luxon'; +import { MicrosoftGraphOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-msgraph'; export default async function createPlugin( @@ -106,53 +104,21 @@ yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-msgraph ): Promise { const builder = await CatalogBuilder.create(env); -+ // The target parameter below needs to match one of the providers' target -+ // value specified in your app-config (see above). + builder.addEntityProvider( + MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { -+ id: 'production', -+ target: 'https://graph.microsoft.com/v1.0', + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ -+ frequency: Duration.fromObject({ minutes: 5 }), -+ timeout: Duration.fromObject({ minutes: 3 }), ++ frequency: { minutes: 5 }, ++ timeout: { minutes: 3 }, + }), + }), + ); ``` -### Using the Processor - -4. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you - have to register it in the catalog plugin: - -```typescript -// packages/backend/src/plugins/catalog.ts -builder.addProcessor( - MicrosoftGraphOrgReaderProcessor.fromConfig(env.config, { - logger: env.logger, - }), -); -``` - -5. Add a location that ingests from Microsoft Graph: - -```yaml -# app-config.yaml -catalog: - locations: - - type: microsoft-graph-org - target: https://graph.microsoft.com/v1.0 - rules: - - allow: [Group, User] - … -``` - ## Customize the Processor or Entity Provider -In case you want to customize the ingested entities, both the `MicrosoftGraphOrgReaderProcessor` -and the `MicrosoftGraphOrgEntityProvider` allows to pass transformers for users, -groups and the organization. +In case you want to customize the ingested entities, the `MicrosoftGraphOrgEntityProvider` +allows to pass transformers for users, groups and the organization. 1. Create a transformer: @@ -179,13 +145,17 @@ export async function myGroupTransformer( } ``` -2. Configure the processor with the transformer: +2. Add the transformer: -```ts -builder.addProcessor( - MicrosoftGraphOrgReaderProcessor.fromConfig(env.config, { - logger: env.logger, - groupTransformer: myGroupTransformer, - }), -); +```diff + builder.addEntityProvider( + MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 5 }, + timeout: { minutes: 3 }, + }), ++ groupTransformer: myGroupTransformer, + }), + ); ``` diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 31a22b0bc4..a4083b1d9e 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -125,14 +125,14 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { static fromConfig( configRoot: Config, options: MicrosoftGraphOrgEntityProviderOptions, - ): MicrosoftGraphOrgEntityProvider; + ): MicrosoftGraphOrgEntityProvider[]; // (undocumented) getProviderName(): string; read(options?: { logger?: Logger }): Promise; } -// @public -export interface MicrosoftGraphOrgEntityProviderOptions { +// @public @deprecated +export interface MicrosoftGraphOrgEntityProviderLegacyOptions { groupTransformer?: GroupTransformer; id: string; logger: Logger; @@ -143,6 +143,19 @@ export interface MicrosoftGraphOrgEntityProviderOptions { } // @public +export type MicrosoftGraphOrgEntityProviderOptions = + | MicrosoftGraphOrgEntityProviderLegacyOptions + | { + logger: Logger; + schedule: 'manual' | TaskRunner; + userTransformer?: UserTransformer | Record; + groupTransformer?: GroupTransformer | Record; + organizationTransformer?: + | OrganizationTransformer + | Record; + }; + +// @public @deprecated export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { constructor(options: { providers: MicrosoftGraphProviderConfig[]; @@ -173,6 +186,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { // @public export type MicrosoftGraphProviderConfig = { + id: string; target: string; authority?: string; tenantId: string; @@ -206,7 +220,7 @@ export type OrganizationTransformer = ( organization: MicrosoftGraph.Organization, ) => Promise; -// @public +// @public @deprecated export function readMicrosoftGraphConfig( config: Config, ): MicrosoftGraphProviderConfig[]; diff --git a/plugins/catalog-backend-module-msgraph/config.d.ts b/plugins/catalog-backend-module-msgraph/config.d.ts index cf00713e72..564ccbb10a 100644 --- a/plugins/catalog-backend-module-msgraph/config.d.ts +++ b/plugins/catalog-backend-module-msgraph/config.d.ts @@ -25,6 +25,7 @@ export interface Config { processors?: { /** * MicrosoftGraphOrgReaderProcessor configuration + * @deprecated Use `catalog.providers.microsoftGraphOrg` instead. */ microsoftGraphOrg?: { /** @@ -102,5 +103,173 @@ export interface Config { }>; }; }; + /** + * List of provider-specific options and attributes + */ + providers?: { + /** + * MicrosoftGraphOrgEntityProvider configuration. + */ + microsoftGraphOrg?: + | { + /** + * The prefix of the target that this matches on, e.g. + * "https://graph.microsoft.com/v1.0", with no trailing slash. + */ + target: string; + /** + * The auth authority used. + * + * Default value "https://login.microsoftonline.com" + */ + authority?: string; + /** + * The tenant whose org data we are interested in. + */ + tenantId: string; + /** + * The OAuth client ID to use for authenticating requests. + */ + clientId: string; + /** + * The OAuth client secret to use for authenticating requests. + * + * @visibility secret + */ + clientSecret: string; + + user?: { + /** + * The "expand" argument to apply to users. + * + * E.g. "manager". + */ + expand?: string; + /** + * The filter to apply to extract users. + * + * E.g. "accountEnabled eq true and userType eq 'member'" + */ + filter?: string; + }; + + group?: { + /** + * The "expand" argument to apply to groups. + * + * E.g. "member". + */ + expand?: string; + /** + * The filter to apply to extract groups. + * + * E.g. "securityEnabled eq false and mailEnabled eq true" + */ + filter?: string; + /** + * The search criteria to apply to extract users by groups memberships. + * + * E.g. "\"displayName:-team\"" would only match groups which contain '-team' + */ + search?: string; + /** + * The fields to be fetched on query. + * + * E.g. ["id", "displayName", "description"] + */ + select?: string[]; + }; + + userGroupMember?: { + /** + * The filter to apply to extract users by groups memberships. + * + * E.g. "displayName eq 'Backstage Users'" + */ + filter?: string; + /** + * The search criteria to apply to extract groups. + * + * E.g. "\"displayName:-team\"" would only match groups which contain '-team' + */ + search?: string; + }; + } + | Record< + string, + { + /** + * The prefix of the target that this matches on, e.g. + * "https://graph.microsoft.com/v1.0", with no trailing slash. + */ + target: string; + /** + * The auth authority used. + * + * Default value "https://login.microsoftonline.com" + */ + authority?: string; + /** + * The tenant whose org data we are interested in. + */ + tenantId: string; + /** + * The OAuth client ID to use for authenticating requests. + */ + clientId: string; + /** + * The OAuth client secret to use for authenticating requests. + * + * @visibility secret + */ + clientSecret: string; + + user?: { + /** + * The filter to apply to extract users. + * + * E.g. "accountEnabled eq true and userType eq 'member'" + */ + filter?: string; + }; + + group?: { + /** + * The filter to apply to extract groups. + * + * E.g. "securityEnabled eq false and mailEnabled eq true" + */ + filter?: string; + /** + * The search criteria to apply to extract users by groups memberships. + * + * E.g. "\"displayName:-team\"" would only match groups which contain '-team' + */ + search?: string; + /** + * The fields to be fetched on query. + * + * E.g. ["id", "displayName", "description"] + */ + select?: string[]; + }; + + userGroupMember?: { + /** + * The filter to apply to extract users by groups memberships. + * + * E.g. "displayName eq 'Backstage Users'" + */ + filter?: string; + /** + * The search criteria to apply to extract groups. + * + * E.g. "\"displayName:-team\"" would only match groups which contain '-team' + */ + search?: string; + }; + } + >; + }; }; } diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index e310fa544f..8239090249 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -15,13 +15,14 @@ */ import { ConfigReader } from '@backstage/config'; -import { readMicrosoftGraphConfig } from './config'; +import { readMicrosoftGraphConfig, readProviderConfigs } from './config'; describe('readMicrosoftGraphConfig', () => { it('applies all of the defaults', () => { const config = { providers: [ { + id: 'target', target: 'target', tenantId: 'tenantId', clientId: 'clientId', @@ -32,6 +33,7 @@ describe('readMicrosoftGraphConfig', () => { const actual = readMicrosoftGraphConfig(new ConfigReader(config)); const expected = [ { + id: 'target', target: 'target', tenantId: 'tenantId', clientId: 'clientId', @@ -48,6 +50,7 @@ describe('readMicrosoftGraphConfig', () => { const config = { providers: [ { + id: 'target', target: 'target', tenantId: 'tenantId', clientId: 'clientId', @@ -64,6 +67,7 @@ describe('readMicrosoftGraphConfig', () => { const actual = readMicrosoftGraphConfig(new ConfigReader(config)); const expected = [ { + id: 'target', target: 'target', tenantId: 'tenantId', clientId: 'clientId', @@ -83,6 +87,7 @@ describe('readMicrosoftGraphConfig', () => { const config = { providers: [ { + id: 'target', target: 'target', tenantId: 'tenantId', clientId: 'clientId', @@ -100,6 +105,7 @@ describe('readMicrosoftGraphConfig', () => { const config = { providers: [ { + id: 'target', target: 'target', tenantId: 'tenantId', clientId: 'clientId', @@ -113,3 +119,128 @@ describe('readMicrosoftGraphConfig', () => { expect(() => readMicrosoftGraphConfig(new ConfigReader(config))).toThrow(); }); }); + +describe('readProviderConfigs', () => { + it('applies all of the defaults', () => { + const config = { + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + }, + }, + }, + }, + }; + const actual = readProviderConfigs(new ConfigReader(config)); + const expected = [ + { + id: 'customProviderId', + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.microsoftonline.com', + }, + ]; + expect(actual).toEqual(expected); + }); + + it('reads all the values', () => { + const config = { + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.example.com/', + user: { + expand: 'manager', + filter: 'accountEnabled eq true', + }, + group: { + expand: 'member', + filter: 'securityEnabled eq false', + select: ['id', 'displayName', 'description'], + }, + }, + }, + }, + }, + }; + const actual = readProviderConfigs(new ConfigReader(config)); + const expected = [ + { + id: 'customProviderId', + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.example.com', + userExpand: 'manager', + userFilter: 'accountEnabled eq true', + groupExpand: 'member', + groupSelect: ['id', 'displayName', 'description'], + groupFilter: 'securityEnabled eq false', + }, + ]; + expect(actual).toEqual(expected); + }); + + it('should fail if both userFilter and userGroupMemberFilter are set', () => { + const config = { + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.example.com/', + user: { + filter: 'accountEnabled eq true', + }, + userGroupMember: { + filter: 'any', + }, + }, + }, + }, + }, + }; + expect(() => readProviderConfigs(new ConfigReader(config))).toThrow(); + }); + + it('should fail if both userFilter and userGroupMemberSearch are set', () => { + const config = { + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.example.com/', + user: { + filter: 'accountEnabled eq true', + }, + userGroupMember: { + search: 'any', + }, + }, + }, + }, + }, + }; + expect(() => readProviderConfigs(new ConfigReader(config))).toThrow(); + }); +}); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index aa2f2d6ee2..30e3a9a4d2 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -17,12 +17,21 @@ import { Config } from '@backstage/config'; import { trimEnd } from 'lodash'; +const DEFAULT_AUTHORITY = 'https://login.microsoftonline.com'; +const DEFAULT_PROVIDER_ID = 'default'; +const DEFAULT_TARGET = 'https://graph.microsoft.com/v1.0'; + /** * The configuration parameters for a single Microsoft Graph provider. * * @public */ export type MicrosoftGraphProviderConfig = { + /** + * Identifier of the provider which will be used i.e. at the location key for ingested entities. + */ + id: string; + /** * The prefix of the target that this matches on, e.g. * "https://graph.microsoft.com/v1.0", with no trailing slash. @@ -55,7 +64,7 @@ export type MicrosoftGraphProviderConfig = { /** * The "expand" argument to apply to users. * - * E.g. "manager" + * E.g. "manager". */ userExpand?: string; /** @@ -73,7 +82,7 @@ export type MicrosoftGraphProviderConfig = { /** * The "expand" argument to apply to groups. * - * E.g. "member" + * E.g. "member". */ groupExpand?: string; /** @@ -113,6 +122,7 @@ export type MicrosoftGraphProviderConfig = { * @param config - The root of the msgraph config hierarchy * * @public + * @deprecated Replaced by not exported `readProviderConfigs` and kept for backwards compatibility only. */ export function readMicrosoftGraphConfig( config: Config, @@ -125,7 +135,7 @@ export function readMicrosoftGraphConfig( const authority = providerConfig.getOptionalString('authority') ? trimEnd(providerConfig.getOptionalString('authority'), '/') - : 'https://login.microsoftonline.com'; + : DEFAULT_AUTHORITY; const tenantId = providerConfig.getString('tenantId'); const clientId = providerConfig.getString('clientId'); const clientSecret = providerConfig.getString('clientSecret'); @@ -164,6 +174,7 @@ export function readMicrosoftGraphConfig( } providers.push({ + id: target, target, authority, tenantId, @@ -183,3 +194,86 @@ export function readMicrosoftGraphConfig( return providers; } + +export function readProviderConfigs( + config: Config, +): MicrosoftGraphProviderConfig[] { + const providersConfig = config.getOptionalConfig( + 'catalog.providers.microsoftGraphOrg', + ); + if (!providersConfig) { + return []; + } + + if (providersConfig.has('clientId')) { + // simple/single config variant + return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; + } + + return providersConfig.keys().map(id => { + const providerConfig = providersConfig.getConfig(id); + + return readProviderConfig(id, providerConfig); + }); +} + +export function readProviderConfig( + id: string, + config: Config, +): MicrosoftGraphProviderConfig { + const target = trimEnd( + config.getOptionalString('target') ?? DEFAULT_TARGET, + '/', + ); + const authority = trimEnd( + config.getOptionalString('authority') ?? DEFAULT_AUTHORITY, + '/', + ); + + const clientId = config.getString('clientId'); + const clientSecret = config.getString('clientSecret'); + const tenantId = config.getString('tenantId'); + + const userExpand = config.getOptionalString('user.expand'); + const userFilter = config.getOptionalString('user.filter'); + + const groupExpand = config.getOptionalString('group.expand'); + const groupFilter = config.getOptionalString('group.filter'); + const groupSearch = config.getOptionalString('group.search'); + const groupSelect = config.getOptionalStringArray('group.select'); + + const userGroupMemberFilter = config.getOptionalString( + 'userGroupMember.filter', + ); + const userGroupMemberSearch = config.getOptionalString( + 'userGroupMember.search', + ); + + if (userFilter && userGroupMemberFilter) { + throw new Error( + `userFilter and userGroupMemberFilter are mutually exclusive, only one can be specified.`, + ); + } + if (userFilter && userGroupMemberSearch) { + throw new Error( + `userGroupMemberSearch cannot be specified when userFilter is defined.`, + ); + } + + return { + id, + target, + authority, + clientId, + clientSecret, + tenantId, + userExpand, + userFilter, + groupExpand, + groupFilter, + groupSearch, + groupSelect, + userGroupMemberFilter, + userGroupMemberSearch, + }; +} diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts index 456afda550..26b4f80765 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -78,19 +79,30 @@ describe('MicrosoftGraphOrgEntityProvider', () => { ], }); + const config = { + catalog: { + providers: { + microsoftGraphOrg: { + customProviderId: { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + }, + }, + }, + }, + }; const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), }; - const provider = new MicrosoftGraphOrgEntityProvider({ - id: 'test', - logger: getVoidLogger(), - provider: { - target: 'https://example.com', - tenantId: 'tenant', - clientId: 'clientid', - clientSecret: 'clientsecret', + const provider = MicrosoftGraphOrgEntityProvider.fromConfig( + new ConfigReader(config), + { + logger: getVoidLogger(), + schedule: 'manual', }, - }); + )[0]; provider.connect(entityProviderConnection); @@ -104,8 +116,10 @@ describe('MicrosoftGraphOrgEntityProvider', () => { kind: 'User', metadata: { annotations: { - 'backstage.io/managed-by-location': 'msgraph:test/u1', - 'backstage.io/managed-by-origin-location': 'msgraph:test/u1', + 'backstage.io/managed-by-location': + 'msgraph:customProviderId/u1', + 'backstage.io/managed-by-origin-location': + 'msgraph:customProviderId/u1', }, name: 'u1', }, @@ -113,7 +127,7 @@ describe('MicrosoftGraphOrgEntityProvider', () => { memberOf: [], }, }, - locationKey: 'msgraph-org-provider:test', + locationKey: 'msgraph-org-provider:customProviderId', }, { entity: { @@ -121,8 +135,10 @@ describe('MicrosoftGraphOrgEntityProvider', () => { kind: 'Group', metadata: { annotations: { - 'backstage.io/managed-by-location': 'msgraph:test/g1', - 'backstage.io/managed-by-origin-location': 'msgraph:test/g1', + 'backstage.io/managed-by-location': + 'msgraph:customProviderId/g1', + 'backstage.io/managed-by-origin-location': + 'msgraph:customProviderId/g1', }, name: 'g1', }, @@ -131,7 +147,7 @@ describe('MicrosoftGraphOrgEntityProvider', () => { type: 'team', }, }, - locationKey: 'msgraph-org-provider:test', + locationKey: 'msgraph-org-provider:customProviderId', }, ], type: 'full', diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index d96faf8e03..4a52ddc665 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -40,13 +40,64 @@ import { readMicrosoftGraphOrg, UserTransformer, } from '../microsoftGraph'; +import { readProviderConfigs } from '../microsoftGraph/config'; /** * Options for {@link MicrosoftGraphOrgEntityProvider}. * * @public */ -export interface MicrosoftGraphOrgEntityProviderOptions { +export type MicrosoftGraphOrgEntityProviderOptions = + | MicrosoftGraphOrgEntityProviderLegacyOptions + | { + /** + * The logger to use. + */ + logger: Logger; + + /** + * The refresh schedule to use. + * + * @remarks + * + * If you pass in 'manual', you are responsible for calling the `read` method + * manually at some interval. + * + * But more commonly you will pass in the result of + * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} + * to enable automatic scheduling of tasks. + */ + schedule: 'manual' | TaskRunner; + + /** + * The function that transforms a user entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + userTransformer?: UserTransformer | Record; + + /** + * The function that transforms a group entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + groupTransformer?: GroupTransformer | Record; + + /** + * The function that transforms an organization entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + organizationTransformer?: + | OrganizationTransformer + | Record; + }; + +/** + * Legacy options for {@link MicrosoftGraphOrgEntityProvider} + * based on `catalog.processors.microsoftGraphOrg`. + * + * @public + * @deprecated This interface exists for backwards compatibility only and will be removed in the future. + */ +export interface MicrosoftGraphOrgEntityProviderLegacyOptions { /** * A unique, stable identifier for this provider. * @@ -57,7 +108,7 @@ export interface MicrosoftGraphOrgEntityProviderOptions { /** * The target that this provider should consume. * - * Should exactly match the "target" field of one of the providers + * Should exactly match the "target" field of one of the provider * configuration entries. */ target: string; @@ -110,7 +161,58 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { static fromConfig( configRoot: Config, options: MicrosoftGraphOrgEntityProviderOptions, - ) { + ): MicrosoftGraphOrgEntityProvider[] { + if ('id' in options) { + return [ + MicrosoftGraphOrgEntityProvider.fromLegacyConfig(configRoot, options), + ]; + } + + function getTransformer( + id: string, + transformers?: T | Record, + ): T | undefined { + if (['undefined', 'function'].includes(typeof transformers)) { + return transformers as T; + } + + return (transformers as Record)[id]; + } + + return readProviderConfigs(configRoot).map(providerConfig => { + const provider = new MicrosoftGraphOrgEntityProvider({ + id: providerConfig.id, + provider: providerConfig, + logger: options.logger, + userTransformer: getTransformer( + providerConfig.id, + options.userTransformer, + ), + groupTransformer: getTransformer( + providerConfig.id, + options.groupTransformer, + ), + organizationTransformer: getTransformer( + providerConfig.id, + options.organizationTransformer, + ), + }); + provider.schedule(options.schedule); + + return provider; + }); + } + + /** + * @deprecated Exists for backwards compatibility only and will be removed in the future. + */ + private static fromLegacyConfig( + configRoot: Config, + options: MicrosoftGraphOrgEntityProviderLegacyOptions, + ): MicrosoftGraphOrgEntityProvider { + options.logger.warn( + 'Deprecated msgraph config "catalog.processors.microsoftGraphOrg" used. Use "catalog.providers.microsoftGraphOrg" instead. More info at https://github.com/backstage/backstage/blob/master/.changeset/long-bananas-rescue.md', + ); const config = configRoot.getOptionalConfig( 'catalog.processors.microsoftGraphOrg', ); diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.test.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.test.ts index 8236dc52e1..bcf76e9482 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.test.ts @@ -38,6 +38,7 @@ describe('MicrosoftGraphOrgReaderProcessor', () => { processor = new MicrosoftGraphOrgReaderProcessor({ providers: [ { + id: 'https://example.com', target: 'https://example.com', tenantId: 'tenant', clientId: 'clientid', diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 5d0e6b2ec7..decd84ec1f 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -33,9 +33,10 @@ import { } from '../microsoftGraph'; /** - * Extracts teams and users out of a the Microsoft Graph API. + * Extracts teams and users out of the Microsoft Graph API. * * @public + * @deprecated Use the MicrosoftGraphOrgEntityProvider instead. */ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { private readonly providers: MicrosoftGraphProviderConfig[]; @@ -67,6 +68,9 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; }) { + options.logger.warn( + 'MicrosoftGraphOrgReaderProcessor is deprecated. Please use MicrosoftGraphOrgEntityProvider instead. More info at https://github.com/backstage/backstage/blob/master/.changeset/long-bananas-rescue.md', + ); this.providers = options.providers; this.logger = options.logger; this.userTransformer = options.userTransformer; diff --git a/plugins/catalog-backend-module-msgraph/src/processors/index.ts b/plugins/catalog-backend-module-msgraph/src/processors/index.ts index 7bc3559c8b..7e74ca1d8b 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/index.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/index.ts @@ -15,5 +15,8 @@ */ export { MicrosoftGraphOrgEntityProvider } from './MicrosoftGraphOrgEntityProvider'; -export type { MicrosoftGraphOrgEntityProviderOptions } from './MicrosoftGraphOrgEntityProvider'; +export type { + MicrosoftGraphOrgEntityProviderOptions, + MicrosoftGraphOrgEntityProviderLegacyOptions, +} from './MicrosoftGraphOrgEntityProvider'; export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';