diff --git a/.changeset/chilly-pumpkins-invent.md b/.changeset/chilly-pumpkins-invent.md new file mode 100644 index 0000000000..19e98d7efb --- /dev/null +++ b/.changeset/chilly-pumpkins-invent.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Allow loading users using group membership diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index 5dedb800f1..d534b7f167 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -56,6 +56,9 @@ catalog: # See https://docs.microsoft.com/en-us/graph/api/resources/user?view=graph-rest-1.0#properties # and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter userFilter: accountEnabled eq true and userType eq 'member' + # Optional filter for users, use group membership to get users + # This appends the users to the list of users retrieved via userFilter + userGroupMemberFilter: "displayName eq 'Backstage Users'" # Optional filter for group, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index db8114eb39..591df9baf7 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -51,6 +51,12 @@ export type MicrosoftGraphProviderConfig = { * E.g. "accountEnabled eq true and userType eq 'member'" */ userFilter?: string; + /** + * The filter to apply to extract users by groups memberships. + * + * E.g. "displayName eq 'Backstage Users'" + */ + userGroupMemberFilter?: string; /** * The filter to apply to extract groups. * @@ -74,6 +80,9 @@ export function readMicrosoftGraphConfig( const clientId = providerConfig.getString('clientId'); const clientSecret = providerConfig.getString('clientSecret'); const userFilter = providerConfig.getOptionalString('userFilter'); + const userGroupMemberFilter = providerConfig.getOptionalString( + 'userGroupMemberFilter', + ); const groupFilter = providerConfig.getOptionalString('groupFilter'); providers.push({ @@ -83,6 +92,7 @@ export function readMicrosoftGraphConfig( clientId, clientSecret, userFilter, + userGroupMemberFilter, groupFilter, }); } diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index 82279b76c6..ffe5146057 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -21,6 +21,7 @@ import { readMicrosoftGraphGroups, readMicrosoftGraphOrganization, readMicrosoftGraphUsers, + readMicrosoftGraphUsersInGroups, resolveRelations, } from './read'; import { getVoidLogger } from '@backstage/backend-common'; @@ -59,6 +60,7 @@ function group(data: Partial): GroupEntity { describe('read microsoft graph', () => { const client: jest.Mocked = { getUsers: jest.fn(), + getUserProfile: jest.fn(), getGroups: jest.fn(), getGroupMembers: jest.fn(), getUserPhotoWithSizeLimit: jest.fn(), @@ -158,6 +160,144 @@ describe('read microsoft graph', () => { }); }); + describe('readMicrosoftGraphUsersInGroups', () => { + it('should read users from Groups', async () => { + async function* getExampleGroups() { + yield { + id: 'groupid', + displayName: 'Group Name', + description: 'Group Description', + mail: 'group@example.com', + }; + } + + async function* getExampleGroupMembers(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.group', + id: 'childgroupid', + }; + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + + client.getUserProfile.mockResolvedValue({ + id: 'userid', + displayName: 'User Name', + mail: 'user.name@example.com', + }); + client.getUserPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { users } = await readMicrosoftGraphUsersInGroups(client, { + userGroupMemberFilter: 'securityEnabled eq true', + logger: getVoidLogger(), + }); + + expect(users).toEqual([ + user({ + metadata: { + annotations: { + 'graph.microsoft.com/user-id': 'userid', + }, + name: 'user.name_example.com', + }, + spec: { + profile: { + displayName: 'User Name', + email: 'user.name@example.com', + picture: 'data:image/jpeg;base64,...', + }, + memberOf: [], + }, + }), + ]); + + expect(client.getGroups).toBeCalledTimes(1); + expect(client.getGroups).toBeCalledWith({ + filter: 'securityEnabled eq true', + }); + expect(client.getGroupMembers).toBeCalledTimes(1); + expect(client.getGroupMembers).toBeCalledWith('groupid'); + + expect(client.getUserProfile).toBeCalledTimes(1); + expect(client.getUserProfile).toBeCalledWith('userid'); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); + }); + + it('should read users with custom transformer', async () => { + async function* getExampleGroups() { + yield { + id: 'groupid', + displayName: 'Group Name', + description: 'Group Description', + mail: 'group@example.com', + }; + } + + async function* getExampleGroupMembers(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.group', + id: 'childgroupid', + }; + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + + client.getUserProfile.mockResolvedValue({ + id: 'userid', + displayName: 'User Name', + mail: 'user.name@example.com', + }); + client.getUserPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { users } = await readMicrosoftGraphUsersInGroups(client, { + userGroupMemberFilter: 'securityEnabled eq true', + transformer: async () => ({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'x' }, + spec: { memberOf: [] }, + }), + logger: getVoidLogger(), + }); + + expect(users).toEqual([ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'x' }, + spec: { memberOf: [] }, + }, + ]); + + expect(client.getGroups).toBeCalledTimes(1); + expect(client.getGroups).toBeCalledWith({ + filter: 'securityEnabled eq true', + }); + expect(client.getGroupMembers).toBeCalledTimes(1); + expect(client.getGroupMembers).toBeCalledWith('groupid'); + + expect(client.getUserProfile).toBeCalledTimes(1); + expect(client.getUserProfile).toBeCalledWith('userid'); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); + }); + }); + describe('readMicrosoftGraphOrganization', () => { it('should read organization', async () => { client.getOrganization.mockResolvedValue({ diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index 096cf8a479..ec4eb22d1e 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -125,6 +125,95 @@ export async function readMicrosoftGraphUsers( return { users }; } +export async function readMicrosoftGraphUsersInGroups( + client: MicrosoftGraphClient, + options: { + userGroupMemberFilter?: string; + transformer?: UserTransformer; + logger: Logger; + }, +): Promise<{ + users: UserEntity[]; // With all relations empty +}> { + const users: UserEntity[] = []; + + if (!options.userGroupMemberFilter) { + return { users }; + } + + const limiter = limiterFactory(10); + + const transformer = options?.transformer ?? defaultUserTransformer; + const userGroupMemberPromises: Promise[] = []; + const userPromises: Promise[] = []; + + const groupMemberUsers: Set = new Set(); + + for await (const group of client.getGroups({ + filter: options?.userGroupMemberFilter, + })) { + // Process all groups in parallel, otherwise it can take quite some time + userGroupMemberPromises.push( + limiter(async () => { + for await (const member of client.getGroupMembers(group.id!)) { + if (!member.id) { + continue; + } + + if (member['@odata.type'] === '#microsoft.graph.user') { + if (!groupMemberUsers.has(member.id)) { + groupMemberUsers.add(member.id); + } + } + } + }), + ); + } + + // Wait for all group members + await Promise.all(userGroupMemberPromises); + + options.logger.info(`groupMemberUsers ${groupMemberUsers.size}`); + for (const userId of groupMemberUsers) { + // Process all users in parallel, otherwise it can take quite some time + userPromises.push( + limiter(async () => { + let user; + let userPhoto; + try { + user = await client.getUserProfile(userId); + } catch (e) { + options.logger.warn(`Unable to load user for ${userId}`); + } + if (user) { + try { + userPhoto = await client.getUserPhotoWithSizeLimit( + user.id!, + // We are limiting the photo size, as users with full resolution photos + // can make the Backstage API slow + 120, + ); + } catch (e) { + options.logger.warn(`Unable to load userphoto for ${userId}`); + } + + const entity = await transformer(user, userPhoto); + + if (!entity) { + return; + } + users.push(entity); + } + }), + ); + } + + // Wait for all users and photos to be downloaded + await Promise.all(userPromises); + + return { users }; +} + export async function defaultOrganizationTransformer( organization: MicrosoftGraph.Organization, ): Promise { @@ -387,6 +476,7 @@ export async function readMicrosoftGraphOrg( tenantId: string, options: { userFilter?: string; + userGroupMemberFilter?: string; groupFilter?: string; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; @@ -394,11 +484,23 @@ export async function readMicrosoftGraphOrg( logger: Logger; }, ): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> { - const { users } = await readMicrosoftGraphUsers(client, { + const { users: usersInGroups } = await readMicrosoftGraphUsersInGroups( + client, + { + userGroupMemberFilter: options.userGroupMemberFilter, + transformer: options.userTransformer, + logger: options.logger, + }, + ); + + const { users: usersWithFilter } = await readMicrosoftGraphUsers(client, { userFilter: options.userFilter, transformer: options.userTransformer, logger: options.logger, }); + + const users: UserEntity[] = usersWithFilter.concat(usersInGroups); + const { groups, rootGroup, groupMember, groupMemberOf } = await readMicrosoftGraphGroups(client, tenantId, { groupFilter: options?.groupFilter, diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 161cc8799f..cb7729e04a 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -101,6 +101,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { provider.tenantId, { userFilter: provider.userFilter, + userGroupMemberFilter: provider.userGroupMemberFilter, groupFilter: provider.groupFilter, userTransformer: this.userTransformer, groupTransformer: this.groupTransformer,