Merge pull request #23048 from fstoerkle/feature/msgraph-ingest-groups-of

feat: add option to ingest group members from Azure Entra ID

.changeset/little-suns-fly.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-msgraph': patch
+---
+
+Added option to ingest groups based on their group membership in Azure Entra ID

docs/integrations/azure/org.md

@@ -111,6 +111,17 @@ microsoftGraphOrg:
       search: '"description:One" AND ("displayName:Video" OR "displayName:Drive")'
 ```
 
+If you don't want to only ingest groups matching the `search` and/or `filter` query, but also the groups which are members of the matched groups, you can use the `includeSubGroups` configuration:
+
+```yaml
+microsoftGraphOrg:
+  providerId:
+    group:
+      filter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified')
+      search: '"description:One" AND ("displayName:Video" OR "displayName:Drive")'
+      includeSubGroups: true
+```
+
 In addition to these groups, one additional group will be created for your organization.
 All imported groups will be a child of this group.
 

plugins/catalog-backend-module-msgraph/api-report.md

@@ -39,10 +39,10 @@ export function defaultUserTransformer(
 // @public
 export type GroupMember =
   | (MicrosoftGraph.Group & {
-      '@odata.type': '#microsoft.graph.user';
+      '@odata.type': '#microsoft.graph.group';
     })
   | (MicrosoftGraph.User & {
-      '@odata.type': '#microsoft.graph.group';
+      '@odata.type': '#microsoft.graph.user';
     });
 
 // @public
@@ -215,6 +215,7 @@ export type MicrosoftGraphProviderConfig = {
   groupFilter?: string;
   groupSearch?: string;
   groupSelect?: string[];
+  groupIncludeSubGroups?: boolean;
   queryMode?: 'basic' | 'advanced';
   loadUserPhotos?: boolean;
   schedule?: TaskScheduleDefinition;
@@ -263,6 +264,7 @@ export function readMicrosoftGraphOrg(
     groupSearch?: string;
     groupFilter?: string;
     groupSelect?: string[];
+    groupIncludeSubGroups?: boolean;
     queryMode?: 'basic' | 'advanced';
     userTransformer?: UserTransformer;
     groupTransformer?: GroupTransformer;

plugins/catalog-backend-module-msgraph/config.d.ts

@@ -192,6 +192,11 @@ export interface Config {
                * E.g. ["id", "displayName", "description"]
                */
               select?: string[];
+              /**
+               * Whether to ingest groups that are members of the found/filtered/searched groups.
+               * Default value is `false`.
+               */
+              includeSubGroups?: boolean;
             };
 
             userGroupMember?: {

plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts

@@ -64,8 +64,8 @@ export type ODataQuery = {
  * @public
  */
 export type GroupMember =
-  | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.user' })
-  | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.group' });
+  | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.group' })
+  | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.user' });
 
 /**
  * A HTTP Client that communicates with Microsoft Graph API.

plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts

@@ -174,6 +174,7 @@ describe('readProviderConfigs', () => {
                 expand: 'member',
                 filter: 'securityEnabled eq false',
                 select: ['id', 'displayName', 'description'],
+                includeSubGroups: true,
               },
               schedule: {
                 frequency: 'PT30M',
@@ -202,6 +203,7 @@ describe('readProviderConfigs', () => {
         groupExpand: 'member',
         groupSelect: ['id', 'displayName', 'description'],
         groupFilter: 'securityEnabled eq false',
+        groupIncludeSubGroups: true,
         schedule: {
           frequency: { minutes: 30 },
           timeout: {

plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts

@@ -116,6 +116,12 @@ export type MicrosoftGraphProviderConfig = {
    */
   groupSelect?: string[];
 
+  /**
+   * Whether to ingest groups that are members of the found/filtered/searched groups.
+   * Default value is `false`.
+   */
+  groupIncludeSubGroups?: boolean;
+
   /**
    * By default, the Microsoft Graph API only provides the basic feature set
    * for querying. Certain features are limited to advanced query capabilities
@@ -292,6 +298,9 @@ export function readProviderConfig(
   const groupFilter = config.getOptionalString('group.filter');
   const groupSearch = config.getOptionalString('group.search');
   const groupSelect = config.getOptionalStringArray('group.select');
+  const groupIncludeSubGroups = config.getOptionalBoolean(
+    'group.includeSubGroups',
+  );
 
   const queryMode = config.getOptionalString('queryMode');
   if (
@@ -347,6 +356,7 @@ export function readProviderConfig(
     groupFilter,
     groupSearch,
     groupSelect,
+    groupIncludeSubGroups,
     queryMode,
     userGroupMemberFilter,
     userGroupMemberSearch,

plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts

@@ -90,6 +90,9 @@ describe('read microsoft graph', () => {
     yield {
       '@odata.type': '#microsoft.graph.group',
       id: 'childgroupid',
+      displayName: 'Child Group Name',
+      description: 'Child Group Description',
+      mail: 'childgroup@example.com',
     };
     yield {
       '@odata.type': '#microsoft.graph.user',
@@ -770,6 +773,117 @@ describe('read microsoft graph', () => {
         top: 999,
       });
     });
+
+    it('should read groups and their sub groups', async () => {
+      async function* getExampleGroupMembersForSubGroup(): AsyncIterable<GroupMember> {
+        yield {
+          '@odata.type': '#microsoft.graph.user',
+          id: 'userid2',
+        };
+      }
+
+      client.getGroups.mockImplementation(getExampleGroups);
+      client.getGroupMembers.mockImplementationOnce(getExampleGroupMembers);
+      client.getGroupMembers.mockImplementationOnce(
+        getExampleGroupMembersForSubGroup,
+      );
+      client.getOrganization.mockResolvedValue({
+        id: 'tenantid',
+        displayName: 'Organization Name',
+      });
+      client.getGroupPhotoWithSizeLimit.mockResolvedValue(
+        'data:image/jpeg;base64,...',
+      );
+
+      const { groups, groupMember, groupMemberOf, rootGroup } =
+        await readMicrosoftGraphGroups(client, 'tenantid', {
+          groupIncludeSubGroups: true,
+        });
+
+      const expectedRootGroup = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/tenant-id': 'tenantid',
+          },
+          name: 'organization_name',
+          description: 'Organization Name',
+        },
+        spec: {
+          type: 'root',
+          profile: {
+            displayName: 'Organization Name',
+          },
+          children: [],
+        },
+      });
+      expect(groups).toEqual([
+        expectedRootGroup,
+        group({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/group-id': 'childgroupid',
+            },
+            name: 'child_group_name',
+            description: 'Child Group Description',
+          },
+          spec: {
+            type: 'team',
+            profile: {
+              displayName: 'Child Group Name',
+              email: 'childgroup@example.com',
+              // TODO: Loading groups photos doesn't work right now as Microsoft
+              // Graph doesn't allows this yet
+              /* picture: 'data:image/jpeg;base64,...',*/
+            },
+            children: [],
+          },
+        }),
+        group({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/group-id': 'groupid',
+            },
+            name: 'group_name',
+            description: 'Group Description',
+          },
+          spec: {
+            type: 'team',
+            profile: {
+              displayName: 'Group Name',
+              email: 'group@example.com',
+              // TODO: Loading groups photos doesn't work right now as Microsoft
+              // Graph doesn't allows this yet
+              /* picture: 'data:image/jpeg;base64,...',*/
+            },
+            children: [],
+          },
+        }),
+      ]);
+      expect(rootGroup).toEqual(expectedRootGroup);
+      expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid']));
+      expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid']));
+      expect(groupMemberOf.get('userid2')).toEqual(new Set(['childgroupid']));
+      expect(groupMember.get('organization_name')).toEqual(new Set());
+
+      expect(client.getGroups).toHaveBeenCalledTimes(1);
+      expect(client.getGroups).toHaveBeenCalledWith(
+        {
+          top: 999,
+        },
+        undefined,
+      );
+      expect(client.getGroupMembers).toHaveBeenCalledTimes(2);
+      expect(client.getGroupMembers).toHaveBeenCalledWith('groupid', {
+        top: 999,
+      });
+      expect(client.getGroupMembers).toHaveBeenCalledWith('childgroupid', {
+        top: 999,
+      });
+      // TODO: Loading groups photos doesn't work right now as Microsoft Graph
+      // doesn't allows this yet
+      // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1);
+      // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120);
+    });
   });
 
   describe('resolveRelations', () => {

plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts

@@ -178,6 +178,7 @@ export async function readMicrosoftGraphGroups(
     groupFilter?: string;
     groupSearch?: string;
     groupSelect?: string[];
+    groupIncludeSubGroups?: boolean;
     groupTransformer?: GroupTransformer;
     organizationTransformer?: OrganizationTransformer;
   },
@@ -244,6 +245,31 @@ export async function readMicrosoftGraphGroups(
 
           if (member['@odata.type'] === '#microsoft.graph.group') {
             ensureItem(groupMember, group.id!, member.id);
+
+            if (options?.groupIncludeSubGroups) {
+              const groupMemberEntity = await transformer(member);
+
+              if (groupMemberEntity) {
+                groups.push(groupMemberEntity);
+
+                for await (const subMember of client.getGroupMembers(
+                  member.id!,
+                  { top: PAGE_SIZE },
+                )) {
+                  if (!subMember.id) {
+                    continue;
+                  }
+
+                  if (subMember['@odata.type'] === '#microsoft.graph.user') {
+                    ensureItem(groupMemberOf, subMember.id, member.id!);
+                  }
+
+                  if (subMember['@odata.type'] === '#microsoft.graph.group') {
+                    ensureItem(groupMember, member.id!, subMember.id);
+                  }
+                }
+              }
+            }
           }
         }
 
@@ -377,6 +403,7 @@ export async function readMicrosoftGraphOrg(
     groupSearch?: string;
     groupFilter?: string;
     groupSelect?: string[];
+    groupIncludeSubGroups?: boolean;
     queryMode?: 'basic' | 'advanced';
     userTransformer?: UserTransformer;
     groupTransformer?: GroupTransformer;
@@ -421,6 +448,7 @@ export async function readMicrosoftGraphOrg(
       groupFilter: options.groupFilter,
       groupSearch: options.groupSearch,
       groupSelect: options.groupSelect,
+      groupIncludeSubGroups: options.groupIncludeSubGroups,
       groupTransformer: options.groupTransformer,
       organizationTransformer: options.organizationTransformer,
     });

plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts

@@ -338,6 +338,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
         groupFilter: provider.groupFilter,
         groupSearch: provider.groupSearch,
         groupSelect: provider.groupSelect,
+        groupIncludeSubGroups: provider.groupIncludeSubGroups,
         queryMode: provider.queryMode,
         groupTransformer: this.options.groupTransformer,
         userTransformer: this.options.userTransformer,