From 95de07271c7d9753f10adcc9b9c4ced1cf1c300f Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Fri, 12 Apr 2024 14:41:59 -0500 Subject: [PATCH 1/5] Updated the Azure and Microsoft Graph Integration Docs Signed-off-by: Andre Wanlin --- docs/integrations/azure/discovery.md | 20 ++++++++++ docs/integrations/azure/org.md | 55 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/docs/integrations/azure/discovery.md b/docs/integrations/azure/discovery.md index 7fbba5228d..4529a31cfe 100644 --- a/docs/integrations/azure/discovery.md +++ b/docs/integrations/azure/discovery.md @@ -127,6 +127,26 @@ builder.addEntityProvider( /* highlight-add-end */ ``` +### New Backend System + +This entity provider will work with the new backend system. Here is how to install it: + +Run the following command to add the module's package: + +```bash +# From your Backstage root directory +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure +``` + +Then updated your backend by adding the following line: + +```ts title="packages/backend/src/index.ts" +backend.add(import('@backstage/plugin-catalog-backend/alpha')); +/* highlight-add-start */ +backend.add(import('@backstage/plugin-catalog-backend-module-azure/alpha')); +/* highlight-add-end */ +``` + ## Alternative Processor As an alternative to the entity provider `AzureDevOpsEntityProvider`, you can still use the `AzureDevopsDiscoveryProcessor`. diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md index 5e39aaeb7d..69151cd82c 100644 --- a/docs/integrations/azure/org.md +++ b/docs/integrations/azure/org.md @@ -64,6 +64,26 @@ export default async function createPlugin( } ``` +### New Backend System + +This entity provider will work with the new backend system. Here is how to install it: + +Run the following command to add the module's package: + +```bash +# From your Backstage root directory +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-msgraph/alpha +``` + +Then updated your backend by adding the following line: + +```ts title="packages/backend/src/index.ts" +backend.add(import('@backstage/plugin-catalog-backend/alpha')); +/* highlight-add-start */ +backend.add(import('@backstage/plugin-catalog-backend-module-msgraph/alpha')); +/* highlight-add-end */ +``` + ## Authenticating with Microsoft Graph ### Local Development @@ -244,6 +264,41 @@ export async function myOrganizationTransformer( } ``` +### Using Transformers in the New Backend System + +Transformers can be configured by extending `microsoftGraphOrgEntityProviderTransformExtensionPoint`. Here is an example: + +```ts title="packages/backend/src/index.ts" +import { createBackendModule } from '@backstage/backend-plugin-api'; +import { microsoftGraphOrgEntityProviderTransformExtensionPoint } from '@backstage/plugin-catalog-backend-module-msgraph/alpha'; + +backend.add( + createBackendModule({ + pluginId: 'catalog', + moduleId: 'microsoft-graph-extensions', + register(env) { + env.registerInit({ + deps: { + /* highlight-add-start */ + microsoftGraphTransformers: + microsoftGraphOrgEntityProviderTransformExtensionPoint, + /* highlight-add-end */ + }, + async init({ microsoftGraphTransformers }) { + /* highlight-add-start */ + microsoftGraphTransformers.setUserTransformer(myUserTransformer); + microsoftGraphTransformers.setGroupTransformer(myGroupTransformer); + microsoftGraphTransformers.setOrganizationTransformer( + myOrganizationTransformer, + ); + /* highlight-add-end */ + }, + }); + }, + }), +); +``` + ## Troubleshooting ### No data From b78ffad266a8fbcb9e9f92996b8a95f2cb3b73a6 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Tue, 30 Apr 2024 09:27:02 -0500 Subject: [PATCH 2/5] Refactored to follow --old convention Signed-off-by: Andre Wanlin --- docs/integrations/azure/discovery--old.md | 191 +++++++++++++++ docs/integrations/azure/discovery.md | 100 +------- docs/integrations/azure/org--old.md | 278 ++++++++++++++++++++++ docs/integrations/azure/org.md | 144 +++++------ 4 files changed, 535 insertions(+), 178 deletions(-) create mode 100644 docs/integrations/azure/discovery--old.md create mode 100644 docs/integrations/azure/org--old.md diff --git a/docs/integrations/azure/discovery--old.md b/docs/integrations/azure/discovery--old.md new file mode 100644 index 0000000000..e9ce137eff --- /dev/null +++ b/docs/integrations/azure/discovery--old.md @@ -0,0 +1,191 @@ +--- +id: discovery--old +title: Azure DevOps Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in an Azure DevOps organization +--- + +:::info +This documentation is written for the old backend which has been replaced by +[the new backend system](../backend-system/index.md), being the default since +Backstage [version 1.24](../releases/v1.24.0.md). If have migrated to the new +backend system, you may want to read [its own article](./discovery.md) +instead. Otherwise, [consider migrating](../backend-system/building-backends/08-migrating.md)! +::: + +The Azure DevOps integration has a special entity provider for discovering +catalog entities within an Azure DevOps. The provider will crawl your Azure +DevOps organization and register entities matching the configured path. This can +be useful as an alternative to static locations or manually adding things to the +catalog. + +This guide explains how to install and configure the Azure DevOps Entity Provider (recommended) or the Azure DevOps Processor. + +## Dependencies + +### Code Search Feature + +Azure discovery is driven by the Code Search feature in Azure DevOps, this may not be enabled by default. For Azure +DevOps Services you can confirm this by looking at the installed extensions in your Organization Settings. For Azure +DevOps Server you'll find this information in your Collection Settings. + +If the Code Search extension is not listed then you can install it from the [Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=ms.vss-code-search&targetId=f9352dac-ba6e-434e-9241-a848a510ce3f&utm_source=vstsproduct&utm_medium=SearchExtStatus). + +### Azure Integration + +Setup [Azure integration](locations.md) with `host` and `token`. Host must be `dev.azure.com` for Cloud users, otherwise set this to your on-premise hostname. + +## Installation + +At your configuration, you add one or more provider configs: + +```yaml title="app-config.yaml" +catalog: + providers: + azureDevOps: + yourProviderId: # identifies your dataset / provider independent of config changes + organization: myorg + project: myproject + repository: service-* # this will match all repos starting with service-* + path: /catalog-info.yaml + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } + yourSecondProviderId: # identifies your dataset / provider independent of config changes + organization: myorg + project: '*' # this will match all projects + repository: '*' # this will match all repos + path: /catalog-info.yaml + anotherProviderId: # another identifier + organization: myorg + project: myproject + repository: '*' # this will match all repos + path: /src/*/catalog-info.yaml # this will search for files deep inside the /src folder + yetAnotherProviderId: # guess, what? Another one :) + host: selfhostedazure.yourcompany.com + organization: myorg + project: myproject + branch: development +``` + +The parameters available are: + +- **`host:`** _(optional)_ Leave empty for Cloud hosted, otherwise set to your self-hosted instance host. +- **`organization:`** Your Organization slug (or Collection for on-premise users). Required. +- **`project:`** _(required)_ Your project slug. Wildcards are supported as shown on the examples above. Using '\*' will search all projects. For a project name containing spaces, use both single and double quotes as in `project: '"My Project Name"'`. +- **`repository:`** _(optional)_ The repository name. Wildcards are supported as show on the examples above. If not set, all repositories will be searched. +- **`path:`** _(optional)_ Where to find catalog-info.yaml files. Defaults to /catalog-info.yaml. +- **`branch:`** _(optional)_ The branch name to use. +- **`schedule`**: + - **`frequency`**: + How often you want the task to run. The system does its best to avoid overlapping invocations. + - **`timeout`**: + The maximum amount of time that a single task invocation can take. + - **`initialDelay`** _(optional)_: + The amount of time that should pass before the first invocation happens. + - **`scope`** _(optional)_: + `'global'` or `'local'`. Sets the scope of concurrency control. + +_Note:_ + +- The path parameter follows the same rules as the search on Azure DevOps web interface. For more details visit the [official search documentation](https://docs.microsoft.com/en-us/azure/devops/project/search/get-started-search?view=azure-devops). +- To use branch parameters, it is necessary that the desired branch be added to the "Searchable branches" list within Azure DevOps Repositories. To do this, follow the instructions below: + +1. Access your Azure DevOps and open the repository in which you want to add the branch. +2. Click on "Settings" in the lower-left corner of the screen. +3. Select the "Options" option in the left navigation bar. +4. In the "Searchable branches" section, click on the "Add" button to add a new branch. +5. In the window that appears, enter the name of the branch you want to add and click "Add". +6. The added branch will now appear in the "Searchable branches" list. + +It may take some time before the branch is indexed and searchable. + +As this provider is not one of the default providers, you will first need to install +the Azure catalog plugin: + +```bash +# From your Backstage root directory +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure +``` + +Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`: + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { AzureDevOpsEntityProvider } from '@backstage/plugin-catalog-backend-module-azure'; + +const builder = await CatalogBuilder.create(env); +/** ... other processors and/or providers ... */ +/* highlight-add-start */ +builder.addEntityProvider( + AzureDevOpsEntityProvider.fromConfig(env.config, { + logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // optional: alternatively, use schedule + scheduler: env.scheduler, + }), +); +/* highlight-add-end */ +``` + +## Alternative Processor + +As an alternative to the entity provider `AzureDevOpsEntityProvider`, you can still use the `AzureDevopsDiscoveryProcessor`. + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { AzureDevOpsDiscoveryProcessor } from '@backstage/plugin-catalog-backend-module-azure'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + /* highlight-add-next-line */ + builder.addProcessor( + AzureDevOpsDiscoveryProcessor.fromConfig(env.config, { + logger: env.logger, + }), + ); + + // .. +} +``` + +```yaml +catalog: + locations: + # Scan all repositories for a catalog-info.yaml in the root of the default branch + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject + # Or use a custom pattern for a subset of all repositories with default repository + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject/_git/service-* + # Or use a custom file format and location + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject/_git/*?path=/src/*/catalog-info.yaml + # And optionally provide a specific branch name using the version parameter + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject/_git/*?path=/catalog-info.yaml&version=GBtopic/catalog-info +``` + +Note the `azure-discovery` type, as this is not a regular `url` processor. + +When using a custom pattern, the target is composed of these parts: + +- The base instance URL, `https://dev.azure.com` in this case +- The organization name which is required, `myorg` in this case +- The project name which is optional, `myproject` in this case. This defaults to \*, which scans all the projects where the token has access to. +- The repository blob to scan, which accepts \* wildcard tokens and must be + added after `_git/`. This can simply be `*` to scan all repositories in the + project. +- The path within each repository to find the catalog YAML file. This will + usually be `/catalog-info.yaml`, `/src/*/catalog-info.yaml` or a similar + variation for catalog files stored in the root directory of each repository. +- The repository branch to scan which is optional, `topic/catalog-info` in this case. If omitted, the repo's default branch will be scanned. The `GB` prefix is required, as this is how Azure DevOps identifies the version as a branch. diff --git a/docs/integrations/azure/discovery.md b/docs/integrations/azure/discovery.md index 4529a31cfe..f63c16a876 100644 --- a/docs/integrations/azure/discovery.md +++ b/docs/integrations/azure/discovery.md @@ -6,6 +6,14 @@ sidebar_label: Discovery description: Automatically discovering catalog entities from repositories in an Azure DevOps organization --- +:::info +This documentation is written for [the new backend +system](../backend-system/index.md) which is the default since Backstage +[version 1.24](../releases/v1.24.0.md). If you are still on the old backend +system, you may want to read [its own article](./discovery--old.md) +instead, and [consider migrating](../backend-system/building-backends/08-migrating.md)! +::: + The Azure DevOps integration has a special entity provider for discovering catalog entities within an Azure DevOps. The provider will crawl your Azure DevOps organization and register entities matching the configured path. This can @@ -30,7 +38,7 @@ Setup [Azure integration](locations.md) with `host` and `token`. Host must be `d ## Installation -At your configuration, you add one or more provider configs: +In your configuration, you add one or more provider configs: ```yaml title="app-config.yaml" catalog: @@ -103,41 +111,6 @@ the Azure catalog plugin: yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure ``` -Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`: - -```ts title="packages/backend/src/plugins/catalog.ts" -/* highlight-add-next-line */ -import { AzureDevOpsEntityProvider } from '@backstage/plugin-catalog-backend-module-azure'; - -const builder = await CatalogBuilder.create(env); -/** ... other processors and/or providers ... */ -/* highlight-add-start */ -builder.addEntityProvider( - AzureDevOpsEntityProvider.fromConfig(env.config, { - logger: env.logger, - // optional: alternatively, use scheduler with schedule defined in app-config.yaml - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 30 }, - timeout: { minutes: 3 }, - }), - // optional: alternatively, use schedule - scheduler: env.scheduler, - }), -); -/* highlight-add-end */ -``` - -### New Backend System - -This entity provider will work with the new backend system. Here is how to install it: - -Run the following command to add the module's package: - -```bash -# From your Backstage root directory -yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure -``` - Then updated your backend by adding the following line: ```ts title="packages/backend/src/index.ts" @@ -146,58 +119,3 @@ backend.add(import('@backstage/plugin-catalog-backend/alpha')); backend.add(import('@backstage/plugin-catalog-backend-module-azure/alpha')); /* highlight-add-end */ ``` - -## Alternative Processor - -As an alternative to the entity provider `AzureDevOpsEntityProvider`, you can still use the `AzureDevopsDiscoveryProcessor`. - -```ts title="packages/backend/src/plugins/catalog.ts" -/* highlight-add-next-line */ -import { AzureDevOpsDiscoveryProcessor } from '@backstage/plugin-catalog-backend-module-azure'; - -export default async function createPlugin( - env: PluginEnvironment, -): Promise { - const builder = await CatalogBuilder.create(env); - /* highlight-add-next-line */ - builder.addProcessor( - AzureDevOpsDiscoveryProcessor.fromConfig(env.config, { - logger: env.logger, - }), - ); - - // .. -} -``` - -```yaml -catalog: - locations: - # Scan all repositories for a catalog-info.yaml in the root of the default branch - - type: azure-discovery - target: https://dev.azure.com/myorg/myproject - # Or use a custom pattern for a subset of all repositories with default repository - - type: azure-discovery - target: https://dev.azure.com/myorg/myproject/_git/service-* - # Or use a custom file format and location - - type: azure-discovery - target: https://dev.azure.com/myorg/myproject/_git/*?path=/src/*/catalog-info.yaml - # And optionally provide a specific branch name using the version parameter - - type: azure-discovery - target: https://dev.azure.com/myorg/myproject/_git/*?path=/catalog-info.yaml&version=GBtopic/catalog-info -``` - -Note the `azure-discovery` type, as this is not a regular `url` processor. - -When using a custom pattern, the target is composed of these parts: - -- The base instance URL, `https://dev.azure.com` in this case -- The organization name which is required, `myorg` in this case -- The project name which is optional, `myproject` in this case. This defaults to \*, which scans all the projects where the token has access to. -- The repository blob to scan, which accepts \* wildcard tokens and must be - added after `_git/`. This can simply be `*` to scan all repositories in the - project. -- The path within each repository to find the catalog YAML file. This will - usually be `/catalog-info.yaml`, `/src/*/catalog-info.yaml` or a similar - variation for catalog files stored in the root directory of each repository. -- The repository branch to scan which is optional, `topic/catalog-info` in this case. If omitted, the repo's default branch will be scanned. The `GB` prefix is required, as this is how Azure DevOps identifies the version as a branch. diff --git a/docs/integrations/azure/org--old.md b/docs/integrations/azure/org--old.md new file mode 100644 index 0000000000..306f8eecb8 --- /dev/null +++ b/docs/integrations/azure/org--old.md @@ -0,0 +1,278 @@ +--- +id: org--old +title: Microsoft Entra Tenant Data +sidebar_label: Org Data +# prettier-ignore +description: Importing users and groups from Microsoft Entra ID into Backstage +--- + +:::info +This documentation is written for the old backend which has been replaced by +[the new backend system](../backend-system/index.md), being the default since +Backstage [version 1.24](../releases/v1.24.0.md). If have migrated to the new +backend system, you may want to read [its own article](./org.md) +instead. Otherwise, [consider migrating](../backend-system/building-backends/08-migrating.md)! +::: + +The Backstage catalog can be set up to ingest organizational data - users and +teams - directly from a tenant in Microsoft Entra ID via the +Microsoft Graph API. + +## Installation + +The package is not installed by default, therefore you have to add `@backstage/plugin-catalog-backend-module-msgraph` to your backend package. + +```bash +# From your Backstage root directory +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-msgraph +``` + +Next add the basic configuration to `app-config.yaml` + +```yaml title="app-config.yaml" +catalog: + providers: + microsoftGraphOrg: + default: + tenantId: ${AZURE_TENANT_ID} + user: + filter: accountEnabled eq true and userType eq 'member' + group: + filter: > + securityEnabled eq false + and mailEnabled eq true + and groupTypes/any(c:c+eq+'Unified') + schedule: + frequency: PT1H + timeout: PT50M +``` + +Finally, register the plugin in `catalog.ts`. +For large organizations, this plugin can take a long time, so be careful setting low frequency / timeouts and importing a large amount of users / groups for the first try. + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { MicrosoftGraphOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-msgraph'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + + /* highlight-add-start */ + builder.addEntityProvider( + MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { + logger: env.logger, + scheduler: env.scheduler, + }), + ); + /* highlight-add-end */ + + // .. +} +``` + +## Authenticating with Microsoft Graph + +### Local Development + +For a local dev environment, it's recommended you have the Azure CLI or Azure PowerShell installed, and are logged in to those. +Alternatively you can use VSCode with the Azure extension if you install `@azure/identity-vscode`. +When these are set up, the plugin will authenticate with the Microsoft Graph API without you needing to configure any credentials, or granting any special permissions. +If you can't do this, you'll have to create an App Registration. + +### App Registration + +If none of the other authentication methods work, you can create an app registration in the azure portal. +By default the graph plugin requires the following Application permissions (not Delegated) for Microsoft Graph: + +- `GroupMember.Read.All` +- `User.Read.All` + +If your organization required Admin Consent for these permissions, that will need to be granted. + +When authenticating with a ClientId/ClientSecret, you can either set the `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET` environment variables, or specify the values in configuration + +```yaml +microsoftGraphOrg: + default: + ##... + clientId: 9ef1aac6-b454-4e69-9cf5-7199df049281 + clientSecret: REDACTED +``` + +To authenticate with a certificate rather than a client secret, you can set the `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_CERTIFICATE_PATH` environments + +### Managed Identity + +If deploying to resources that supports Managed Identity, and has identities configured (e.g. Azure App Services, Azure Container Apps), Managed Identity should be picked up without any additional configuration. +If your app has multiple managed identities, you may need to set the `AZURE_CLIENT_ID` environment variable to tell Azure Identity which identity to use. + +To grant the managed identity the same permissions as mentioned in _App Registration_ above, [please follow this guide](https://docs.microsoft.com/en-us/azure/app-service/tutorial-connect-app-access-microsoft-graph-as-app-javascript?tabs=azure-powershell) + +## Filtering imported Users and Groups + +By default, the plugin will import all users and groups from your directory. +This can be customized through [filters](https://learn.microsoft.com/en-us/graph/filter-query-parameter) and [search](https://learn.microsoft.com/en-us/graph/search-query-parameter) queries. Keep in mind that if you omit filters and search queries for the user or group properties, the plugin will automatically import all available users or groups. + +### Groups + +A smaller set of groups can be obtained by configuring a search query or a filter. +If both `filter` and `search` are provided, then groups must match both to be ingested. + +```yaml +microsoftGraphOrg: + providerId: + group: + filter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') + search: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' +``` + +In addition to these groups, one additional group will be created for your organization. +All imported groups will be a child of this group. + +### Users + +There are two modes for importing users - You can import all user objects matching a `filter`. + +```yaml +microsoftGraphOrg: + providerId: + user: + filter: accountEnabled eq true and userType eq 'member' +``` + +Alternatively you can import users that are members of specific groups. +For each group matching the `search` and `filter` query, each group member will be imported. +Only direct group members will be imported, not transient users. + +```yaml +microsoftGraphOrg: + providerId: + userGroupMember: + filter: "displayName eq 'Backstage Users'" + search: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' +``` + +### User photos + +By default, the photos of users will be fetched and added to each user entity. For huge organizations this may be unfeasible, as it will take a _very_ long time, and can be disabled by setting `loadPhotos` to `false`: + +```yaml +microsoftGraphOrg: + providerId: + user: + filter: ... + loadPhotos: false +``` + +## Customizing Transformation + +Ingested entities can be customized by providing custom transformers. +These can be used to completely replace the built in logic, or used to tweak it by using the default transformers (`defaultGroupTransformer`, `defaultUserTransformer` and `defaultOrganizationTransformer` +Entities can also be excluded from backstage by returning `undefined`. + +These Transformers are be registered when configuring `MicrosoftGraphOrgEntityProvider` + +```ts +builder.addEntityProvider( + MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { + // ... + /* highlight-add-start */ + groupTransformer: myGroupTransformer, + userTransformer: myUserTransformer, + organizationTransformer: myOrganizationTransformer, + /* highlight-add-end */ + }), +); +``` + +When using custom transformers, you may want to customize the data returned. +Several configuration options can be provided to tweak the Microsoft Graph query to get the data you need + +```yaml +microsoftGraphOrg: + providerId: + user: + expand: manager + group: + expand: member + select: ['id', 'displayName', 'description'] +``` + +The following provides an example of each kind of transformer + +```ts +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; +import { + defaultGroupTransformer, + defaultUserTransformer, + defaultOrganizationTransformer, +} from '@backstage/plugin-catalog-backend-module-msgraph'; +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; + +// This group transformer completely replaces the built in logic with custom logic. +export async function myGroupTransformer( + group: MicrosoftGraph.Group, + groupPhoto?: string, +): Promise { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: group.id!, + annotations: {}, + }, + spec: { + type: 'Microsoft Entra ID', + children: [], + }, + }; +} + +// This user transformer makes use of the built in logic, but also sets the description field +export async function myUserTransformer( + graphUser: MicrosoftGraph.User, + userPhoto?: string, +): Promise { + const backstageUser = await defaultUserTransformer(graphUser, userPhoto); + + if (backstageUser) { + backstageUser.metadata.description = 'Loaded from Microsoft Entra ID'; + } + + return backstageUser; +} + +// Example organization transformer that removes the organization group completely +export async function myOrganizationTransformer( + graphOrganization: MicrosoftGraph.Organization, +): Promise { + return undefined; +} +``` + +## Troubleshooting + +### No data + +First check your logs for the message `Reading msgraph users and groups`. +If you don't see this, check you've registered the provider, and that the schedule is valid + +If you see a log entry `Read 0 msgraph users and 0 msgraph groups`, check your search and filter arguments. + +If you see the start message (`Reading msgraph users and groups`) but no end message (`Read X msgraph users and Y msgraph groups`), then it is likely the job is taking a long time due to a large volume of data. +The default behavior is to import all users and groups, which is often more data than needed. +Try importing a smaller set of data (e.g. `filter: displayName eq 'John Smith'`). + +### Authentication / Token Errors + +See [Troubleshooting Azure Identity Authentication Issues](https://aka.ms/azsdk/js/identity/troubleshoot) + +### Error while reading users from Microsoft Graph: Authorization_RequestDenied - Insufficient privileges to complete the operation + +- Make sure you've granted all the required permissions to your application registration or managed identity +- Make sure the permissions are `Application` permissions rather than `Delegated` +- If your organization has configured "Admin consent" to be required, make sure this has been granted for your application permissions +- If your group queries are returning Microsoft Teams groups, you may need to grant addition permissions (e.g. `Team.ReadBasic.All`, `TeamMember.Read.All`) +- If you've added additional `select` or `expand` fields, those may need additional permissions granted diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md index 69151cd82c..46623a7537 100644 --- a/docs/integrations/azure/org.md +++ b/docs/integrations/azure/org.md @@ -6,6 +6,14 @@ sidebar_label: Org Data description: Importing users and groups from Microsoft Entra ID into Backstage --- +:::info +This documentation is written for [the new backend +system](../backend-system/index.md) which is the default since Backstage +[version 1.24](../releases/v1.24.0.md). If you are still on the old backend +system, you may want to read [its own article](./identity-resolver--old.md) +instead, and [consider migrating](../backend-system/building-backends/08-migrating.md)! +::: + The Backstage catalog can be set up to ingest organizational data - users and teams - directly from a tenant in Microsoft Entra ID via the Microsoft Graph API. @@ -39,43 +47,11 @@ catalog: timeout: PT50M ``` -Finally, register the plugin in `catalog.ts`. +:::note For large organizations, this plugin can take a long time, so be careful setting low frequency / timeouts and importing a large amount of users / groups for the first try. +::: -```ts title="packages/backend/src/plugins/catalog.ts" -/* highlight-add-next-line */ -import { MicrosoftGraphOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-msgraph'; - -export default async function createPlugin( - env: PluginEnvironment, -): Promise { - const builder = await CatalogBuilder.create(env); - - /* highlight-add-start */ - builder.addEntityProvider( - MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { - logger: env.logger, - scheduler: env.scheduler, - }), - ); - /* highlight-add-end */ - - // .. -} -``` - -### New Backend System - -This entity provider will work with the new backend system. Here is how to install it: - -Run the following command to add the module's package: - -```bash -# From your Backstage root directory -yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-msgraph/alpha -``` - -Then updated your backend by adding the following line: +Finally, updated your backend by adding the following line: ```ts title="packages/backend/src/index.ts" backend.add(import('@backstage/plugin-catalog-backend/alpha')); @@ -184,21 +160,6 @@ Ingested entities can be customized by providing custom transformers. These can be used to completely replace the built in logic, or used to tweak it by using the default transformers (`defaultGroupTransformer`, `defaultUserTransformer` and `defaultOrganizationTransformer` Entities can also be excluded from backstage by returning `undefined`. -These Transformers are be registered when configuring `MicrosoftGraphOrgEntityProvider` - -```ts -builder.addEntityProvider( - MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { - // ... - /* highlight-add-start */ - groupTransformer: myGroupTransformer, - userTransformer: myUserTransformer, - organizationTransformer: myOrganizationTransformer, - /* highlight-add-end */ - }), -); -``` - When using custom transformers, you may want to customize the data returned. Several configuration options can be provided to tweak the Microsoft Graph query to get the data you need @@ -212,9 +173,53 @@ microsoftGraphOrg: select: ['id', 'displayName', 'description'] ``` -The following provides an example of each kind of transformer +### Using Custom Transformers -```ts +Transformers can be configured by extending `microsoftGraphOrgEntityProviderTransformExtensionPoint`. Here is an example: + +```ts title="packages/backend/src/index.ts" +import { createBackendModule } from '@backstage/backend-plugin-api'; +import { microsoftGraphOrgEntityProviderTransformExtensionPoint } from '@backstage/plugin-catalog-backend-module-msgraph/alpha'; +import { + myUserTransformer, + myGroupTransformer, + myOrganizationTransformer, +} from './transformers'; + +backend.add( + createBackendModule({ + pluginId: 'catalog', + moduleId: 'microsoft-graph-extensions', + register(env) { + env.registerInit({ + deps: { + /* highlight-add-start */ + microsoftGraphTransformers: + microsoftGraphOrgEntityProviderTransformExtensionPoint, + /* highlight-add-end */ + }, + async init({ microsoftGraphTransformers }) { + /* highlight-add-start */ + microsoftGraphTransformers.setUserTransformer(myUserTransformer); + microsoftGraphTransformers.setGroupTransformer(myGroupTransformer); + microsoftGraphTransformers.setOrganizationTransformer( + myOrganizationTransformer, + ); + /* highlight-add-end */ + }, + }); + }, + }), +); +``` + +The `myUserTransformer`, `myGroupTransformer`, and `myOrganizationTransformer` transformer functions are from the examples in the section below. + +### Transformer Examples + +The following provides an example of each kind of transformer. We recommend creating a `transformers.ts` file in your `packages/backend/src` folder for these. + +```ts title="packages/backend/src/transformers.ts" import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; import { defaultGroupTransformer, @@ -264,41 +269,6 @@ export async function myOrganizationTransformer( } ``` -### Using Transformers in the New Backend System - -Transformers can be configured by extending `microsoftGraphOrgEntityProviderTransformExtensionPoint`. Here is an example: - -```ts title="packages/backend/src/index.ts" -import { createBackendModule } from '@backstage/backend-plugin-api'; -import { microsoftGraphOrgEntityProviderTransformExtensionPoint } from '@backstage/plugin-catalog-backend-module-msgraph/alpha'; - -backend.add( - createBackendModule({ - pluginId: 'catalog', - moduleId: 'microsoft-graph-extensions', - register(env) { - env.registerInit({ - deps: { - /* highlight-add-start */ - microsoftGraphTransformers: - microsoftGraphOrgEntityProviderTransformExtensionPoint, - /* highlight-add-end */ - }, - async init({ microsoftGraphTransformers }) { - /* highlight-add-start */ - microsoftGraphTransformers.setUserTransformer(myUserTransformer); - microsoftGraphTransformers.setGroupTransformer(myGroupTransformer); - microsoftGraphTransformers.setOrganizationTransformer( - myOrganizationTransformer, - ); - /* highlight-add-end */ - }, - }); - }, - }), -); -``` - ## Troubleshooting ### No data From 8450ccd204580063a6152bb197c79e06fd0284ee Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Tue, 30 Apr 2024 12:08:15 -0500 Subject: [PATCH 3/5] Fixed links Signed-off-by: Andre Wanlin --- docs/integrations/azure/discovery--old.md | 6 +----- docs/integrations/azure/discovery.md | 6 +----- docs/integrations/azure/org--old.md | 6 +----- docs/integrations/azure/org.md | 6 +----- 4 files changed, 4 insertions(+), 20 deletions(-) diff --git a/docs/integrations/azure/discovery--old.md b/docs/integrations/azure/discovery--old.md index e9ce137eff..be490d70fe 100644 --- a/docs/integrations/azure/discovery--old.md +++ b/docs/integrations/azure/discovery--old.md @@ -7,11 +7,7 @@ description: Automatically discovering catalog entities from repositories in an --- :::info -This documentation is written for the old backend which has been replaced by -[the new backend system](../backend-system/index.md), being the default since -Backstage [version 1.24](../releases/v1.24.0.md). If have migrated to the new -backend system, you may want to read [its own article](./discovery.md) -instead. Otherwise, [consider migrating](../backend-system/building-backends/08-migrating.md)! +This documentation is written for the old backend which has been replaced by [the new backend system](../../backend-system/index.md), being the default since Backstage [version 1.24](../../releases/v1.24.0.md). If have migrated to the new backend system, you may want to read [its own article](./discovery.md) instead. Otherwise, [consider migrating](../../backend-system/building-backends/08-migrating.md)! ::: The Azure DevOps integration has a special entity provider for discovering diff --git a/docs/integrations/azure/discovery.md b/docs/integrations/azure/discovery.md index f63c16a876..00c2cca104 100644 --- a/docs/integrations/azure/discovery.md +++ b/docs/integrations/azure/discovery.md @@ -7,11 +7,7 @@ description: Automatically discovering catalog entities from repositories in an --- :::info -This documentation is written for [the new backend -system](../backend-system/index.md) which is the default since Backstage -[version 1.24](../releases/v1.24.0.md). If you are still on the old backend -system, you may want to read [its own article](./discovery--old.md) -instead, and [consider migrating](../backend-system/building-backends/08-migrating.md)! +This documentation is written for [the new backend system](../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If you are still on the old backend system, you may want to read [its own article](./discovery--old.md) instead, and [consider migrating](../../backend-system/building-backends/08-migrating.md)! ::: The Azure DevOps integration has a special entity provider for discovering diff --git a/docs/integrations/azure/org--old.md b/docs/integrations/azure/org--old.md index 306f8eecb8..89d1dfd61d 100644 --- a/docs/integrations/azure/org--old.md +++ b/docs/integrations/azure/org--old.md @@ -7,11 +7,7 @@ description: Importing users and groups from Microsoft Entra ID into Backstage --- :::info -This documentation is written for the old backend which has been replaced by -[the new backend system](../backend-system/index.md), being the default since -Backstage [version 1.24](../releases/v1.24.0.md). If have migrated to the new -backend system, you may want to read [its own article](./org.md) -instead. Otherwise, [consider migrating](../backend-system/building-backends/08-migrating.md)! +This documentation is written for the old backend which has been replaced by [the new backend system](..././backend-system/index.md), being the default since Backstage [version 1.24](../../releases/v1.24.0.md). If have migrated to the new backend system, you may want to read [its own article](./org.md) instead.Otherwise, [consider migrating](../../backend-system/building-backends/08-migrating.md)! ::: The Backstage catalog can be set up to ingest organizational data - users and diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md index 46623a7537..89abd4813b 100644 --- a/docs/integrations/azure/org.md +++ b/docs/integrations/azure/org.md @@ -7,11 +7,7 @@ description: Importing users and groups from Microsoft Entra ID into Backstage --- :::info -This documentation is written for [the new backend -system](../backend-system/index.md) which is the default since Backstage -[version 1.24](../releases/v1.24.0.md). If you are still on the old backend -system, you may want to read [its own article](./identity-resolver--old.md) -instead, and [consider migrating](../backend-system/building-backends/08-migrating.md)! +This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If you are still on the old backend system, you may want to read [its own article](./org--old.md) instead, and [consider migrating](../../backend-system/building-backends/08-migrating.md)! ::: The Backstage catalog can be set up to ingest organizational data - users and From 353880c4501cbbffee657c89deb50e5d115b111c Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Tue, 30 Apr 2024 12:52:41 -0500 Subject: [PATCH 4/5] Fixed missing links Signed-off-by: Andre Wanlin --- docs/integrations/azure/discovery.md | 2 +- docs/integrations/azure/org--old.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/azure/discovery.md b/docs/integrations/azure/discovery.md index 00c2cca104..7e7b465118 100644 --- a/docs/integrations/azure/discovery.md +++ b/docs/integrations/azure/discovery.md @@ -7,7 +7,7 @@ description: Automatically discovering catalog entities from repositories in an --- :::info -This documentation is written for [the new backend system](../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If you are still on the old backend system, you may want to read [its own article](./discovery--old.md) instead, and [consider migrating](../../backend-system/building-backends/08-migrating.md)! +This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If you are still on the old backend system, you may want to read [its own article](./discovery--old.md) instead, and [consider migrating](../../backend-system/building-backends/08-migrating.md)! ::: The Azure DevOps integration has a special entity provider for discovering diff --git a/docs/integrations/azure/org--old.md b/docs/integrations/azure/org--old.md index 89d1dfd61d..9f4b70db26 100644 --- a/docs/integrations/azure/org--old.md +++ b/docs/integrations/azure/org--old.md @@ -7,7 +7,7 @@ description: Importing users and groups from Microsoft Entra ID into Backstage --- :::info -This documentation is written for the old backend which has been replaced by [the new backend system](..././backend-system/index.md), being the default since Backstage [version 1.24](../../releases/v1.24.0.md). If have migrated to the new backend system, you may want to read [its own article](./org.md) instead.Otherwise, [consider migrating](../../backend-system/building-backends/08-migrating.md)! +This documentation is written for the old backend which has been replaced by [the new backend system](.../../backend-system/index.md), being the default since Backstage [version 1.24](../../releases/v1.24.0.md). If have migrated to the new backend system, you may want to read [its own article](./org.md) instead.Otherwise, [consider migrating](../../backend-system/building-backends/08-migrating.md)! ::: The Backstage catalog can be set up to ingest organizational data - users and From b0eb7588ddee55aa63172eab6a1a4f7f32d61c4b Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Tue, 30 Apr 2024 13:04:09 -0500 Subject: [PATCH 5/5] Fixed link typo Signed-off-by: Andre Wanlin --- docs/integrations/azure/org--old.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/azure/org--old.md b/docs/integrations/azure/org--old.md index 9f4b70db26..13d53b24b3 100644 --- a/docs/integrations/azure/org--old.md +++ b/docs/integrations/azure/org--old.md @@ -7,7 +7,7 @@ description: Importing users and groups from Microsoft Entra ID into Backstage --- :::info -This documentation is written for the old backend which has been replaced by [the new backend system](.../../backend-system/index.md), being the default since Backstage [version 1.24](../../releases/v1.24.0.md). If have migrated to the new backend system, you may want to read [its own article](./org.md) instead.Otherwise, [consider migrating](../../backend-system/building-backends/08-migrating.md)! +This documentation is written for the old backend which has been replaced by [the new backend system](../../backend-system/index.md), being the default since Backstage [version 1.24](../../releases/v1.24.0.md). If have migrated to the new backend system, you may want to read [its own article](./org.md) instead.Otherwise, [consider migrating](../../backend-system/building-backends/08-migrating.md)! ::: The Backstage catalog can be set up to ingest organizational data - users and