Allow loading users using group membership

The userFilter loads users by filtering on their attributes only.
This allows to load users based on group membership e.g. for cases
where only a certain set of users should be loaded using their
group membership

Signed-off-by: Mustansar Anwar ul Samad <mustansar.samad@gmail.com>

.changeset/chilly-pumpkins-invent.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-msgraph': patch
+---
+
+Allow loading users using group membership

plugins/catalog-backend-module-msgraph/README.md

@@ -56,6 +56,9 @@ catalog:
           # See https://docs.microsoft.com/en-us/graph/api/resources/user?view=graph-rest-1.0#properties
           # and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter
           userFilter: accountEnabled eq true and userType eq 'member'
+          # Optional filter for users, use group membership to get users
+          # This appends the users to the list of users retrieved via userFilter
+          userGroupMemberFilter: "displayName eq 'Backstage Users'"
           # Optional filter for group, see Microsoft Graph API for the syntax
           # See https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties
           groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified')

plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts

@@ -51,6 +51,12 @@ export type MicrosoftGraphProviderConfig = {
    * E.g. "accountEnabled eq true and userType eq 'member'"
    */
   userFilter?: string;
+  /**
+   * The filter to apply to extract users by groups memberships.
+   *
+   * E.g. "displayName eq 'Backstage Users'"
+   */
+  userGroupMemberFilter?: string;
   /**
    * The filter to apply to extract groups.
    *
@@ -74,6 +80,9 @@ export function readMicrosoftGraphConfig(
     const clientId = providerConfig.getString('clientId');
     const clientSecret = providerConfig.getString('clientSecret');
     const userFilter = providerConfig.getOptionalString('userFilter');
+    const userGroupMemberFilter = providerConfig.getOptionalString(
+      'userGroupMemberFilter',
+    );
     const groupFilter = providerConfig.getOptionalString('groupFilter');
 
     providers.push({
@@ -83,6 +92,7 @@ export function readMicrosoftGraphConfig(
       clientId,
       clientSecret,
       userFilter,
+      userGroupMemberFilter,
       groupFilter,
     });
   }

plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts

@@ -21,6 +21,7 @@ import {
   readMicrosoftGraphGroups,
   readMicrosoftGraphOrganization,
   readMicrosoftGraphUsers,
+  readMicrosoftGraphUsersInGroups,
   resolveRelations,
 } from './read';
 import { getVoidLogger } from '@backstage/backend-common';
@@ -59,6 +60,7 @@ function group(data: Partial<GroupEntity>): GroupEntity {
 describe('read microsoft graph', () => {
   const client: jest.Mocked<MicrosoftGraphClient> = {
     getUsers: jest.fn(),
+    getUserProfile: jest.fn(),
     getGroups: jest.fn(),
     getGroupMembers: jest.fn(),
     getUserPhotoWithSizeLimit: jest.fn(),
@@ -158,6 +160,144 @@ describe('read microsoft graph', () => {
     });
   });
 
+  describe('readMicrosoftGraphUsersInGroups', () => {
+    it('should read users from Groups', async () => {
+      async function* getExampleGroups() {
+        yield {
+          id: 'groupid',
+          displayName: 'Group Name',
+          description: 'Group Description',
+          mail: 'group@example.com',
+        };
+      }
+
+      async function* getExampleGroupMembers(): AsyncIterable<GroupMember> {
+        yield {
+          '@odata.type': '#microsoft.graph.group',
+          id: 'childgroupid',
+        };
+        yield {
+          '@odata.type': '#microsoft.graph.user',
+          id: 'userid',
+        };
+      }
+
+      client.getGroups.mockImplementation(getExampleGroups);
+      client.getGroupMembers.mockImplementation(getExampleGroupMembers);
+
+      client.getUserProfile.mockResolvedValue({
+        id: 'userid',
+        displayName: 'User Name',
+        mail: 'user.name@example.com',
+      });
+      client.getUserPhotoWithSizeLimit.mockResolvedValue(
+        'data:image/jpeg;base64,...',
+      );
+
+      const { users } = await readMicrosoftGraphUsersInGroups(client, {
+        userGroupMemberFilter: 'securityEnabled eq true',
+        logger: getVoidLogger(),
+      });
+
+      expect(users).toEqual([
+        user({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/user-id': 'userid',
+            },
+            name: 'user.name_example.com',
+          },
+          spec: {
+            profile: {
+              displayName: 'User Name',
+              email: 'user.name@example.com',
+              picture: 'data:image/jpeg;base64,...',
+            },
+            memberOf: [],
+          },
+        }),
+      ]);
+
+      expect(client.getGroups).toBeCalledTimes(1);
+      expect(client.getGroups).toBeCalledWith({
+        filter: 'securityEnabled eq true',
+      });
+      expect(client.getGroupMembers).toBeCalledTimes(1);
+      expect(client.getGroupMembers).toBeCalledWith('groupid');
+
+      expect(client.getUserProfile).toBeCalledTimes(1);
+      expect(client.getUserProfile).toBeCalledWith('userid');
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1);
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120);
+    });
+
+    it('should read users with custom transformer', async () => {
+      async function* getExampleGroups() {
+        yield {
+          id: 'groupid',
+          displayName: 'Group Name',
+          description: 'Group Description',
+          mail: 'group@example.com',
+        };
+      }
+
+      async function* getExampleGroupMembers(): AsyncIterable<GroupMember> {
+        yield {
+          '@odata.type': '#microsoft.graph.group',
+          id: 'childgroupid',
+        };
+        yield {
+          '@odata.type': '#microsoft.graph.user',
+          id: 'userid',
+        };
+      }
+
+      client.getGroups.mockImplementation(getExampleGroups);
+      client.getGroupMembers.mockImplementation(getExampleGroupMembers);
+
+      client.getUserProfile.mockResolvedValue({
+        id: 'userid',
+        displayName: 'User Name',
+        mail: 'user.name@example.com',
+      });
+      client.getUserPhotoWithSizeLimit.mockResolvedValue(
+        'data:image/jpeg;base64,...',
+      );
+
+      const { users } = await readMicrosoftGraphUsersInGroups(client, {
+        userGroupMemberFilter: 'securityEnabled eq true',
+        transformer: async () => ({
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'User',
+          metadata: { name: 'x' },
+          spec: { memberOf: [] },
+        }),
+        logger: getVoidLogger(),
+      });
+
+      expect(users).toEqual([
+        {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'User',
+          metadata: { name: 'x' },
+          spec: { memberOf: [] },
+        },
+      ]);
+
+      expect(client.getGroups).toBeCalledTimes(1);
+      expect(client.getGroups).toBeCalledWith({
+        filter: 'securityEnabled eq true',
+      });
+      expect(client.getGroupMembers).toBeCalledTimes(1);
+      expect(client.getGroupMembers).toBeCalledWith('groupid');
+
+      expect(client.getUserProfile).toBeCalledTimes(1);
+      expect(client.getUserProfile).toBeCalledWith('userid');
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1);
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120);
+    });
+  });
+
   describe('readMicrosoftGraphOrganization', () => {
     it('should read organization', async () => {
       client.getOrganization.mockResolvedValue({

plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts

@@ -125,6 +125,95 @@ export async function readMicrosoftGraphUsers(
   return { users };
 }
 
+export async function readMicrosoftGraphUsersInGroups(
+  client: MicrosoftGraphClient,
+  options: {
+    userGroupMemberFilter?: string;
+    transformer?: UserTransformer;
+    logger: Logger;
+  },
+): Promise<{
+  users: UserEntity[]; // With all relations empty
+}> {
+  const users: UserEntity[] = [];
+
+  if (!options.userGroupMemberFilter) {
+    return { users };
+  }
+
+  const limiter = limiterFactory(10);
+
+  const transformer = options?.transformer ?? defaultUserTransformer;
+  const userGroupMemberPromises: Promise<void>[] = [];
+  const userPromises: Promise<void>[] = [];
+
+  const groupMemberUsers: Set<string> = new Set();
+
+  for await (const group of client.getGroups({
+    filter: options?.userGroupMemberFilter,
+  })) {
+    // Process all groups in parallel, otherwise it can take quite some time
+    userGroupMemberPromises.push(
+      limiter(async () => {
+        for await (const member of client.getGroupMembers(group.id!)) {
+          if (!member.id) {
+            continue;
+          }
+
+          if (member['@odata.type'] === '#microsoft.graph.user') {
+            if (!groupMemberUsers.has(member.id)) {
+              groupMemberUsers.add(member.id);
+            }
+          }
+        }
+      }),
+    );
+  }
+
+  // Wait for all group members
+  await Promise.all(userGroupMemberPromises);
+
+  options.logger.info(`groupMemberUsers ${groupMemberUsers.size}`);
+  for (const userId of groupMemberUsers) {
+    // Process all users in parallel, otherwise it can take quite some time
+    userPromises.push(
+      limiter(async () => {
+        let user;
+        let userPhoto;
+        try {
+          user = await client.getUserProfile(userId);
+        } catch (e) {
+          options.logger.warn(`Unable to load user for ${userId}`);
+        }
+        if (user) {
+          try {
+            userPhoto = await client.getUserPhotoWithSizeLimit(
+              user.id!,
+              // We are limiting the photo size, as users with full resolution photos
+              // can make the Backstage API slow
+              120,
+            );
+          } catch (e) {
+            options.logger.warn(`Unable to load userphoto for ${userId}`);
+          }
+
+          const entity = await transformer(user, userPhoto);
+
+          if (!entity) {
+            return;
+          }
+          users.push(entity);
+        }
+      }),
+    );
+  }
+
+  // Wait for all users and photos to be downloaded
+  await Promise.all(userPromises);
+
+  return { users };
+}
+
 export async function defaultOrganizationTransformer(
   organization: MicrosoftGraph.Organization,
 ): Promise<GroupEntity | undefined> {
@@ -387,6 +476,7 @@ export async function readMicrosoftGraphOrg(
   tenantId: string,
   options: {
     userFilter?: string;
+    userGroupMemberFilter?: string;
     groupFilter?: string;
     userTransformer?: UserTransformer;
     groupTransformer?: GroupTransformer;
@@ -394,11 +484,23 @@ export async function readMicrosoftGraphOrg(
     logger: Logger;
   },
 ): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> {
-  const { users } = await readMicrosoftGraphUsers(client, {
+  const { users: usersInGroups } = await readMicrosoftGraphUsersInGroups(
+    client,
+    {
+      userGroupMemberFilter: options.userGroupMemberFilter,
+      transformer: options.userTransformer,
+      logger: options.logger,
+    },
+  );
+
+  const { users: usersWithFilter } = await readMicrosoftGraphUsers(client, {
     userFilter: options.userFilter,
     transformer: options.userTransformer,
     logger: options.logger,
   });
+
+  const users: UserEntity[] = usersWithFilter.concat(usersInGroups);
+
   const { groups, rootGroup, groupMember, groupMemberOf } =
     await readMicrosoftGraphGroups(client, tenantId, {
       groupFilter: options?.groupFilter,

plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts

@@ -101,6 +101,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
       provider.tenantId,
       {
         userFilter: provider.userFilter,
+        userGroupMemberFilter: provider.userGroupMemberFilter,
         groupFilter: provider.groupFilter,
         userTransformer: this.userTransformer,
         groupTransformer: this.groupTransformer,