diff --git a/.changeset/beige-lamps-cry.md b/.changeset/beige-lamps-cry.md new file mode 100644 index 0000000000..c77d5689c5 --- /dev/null +++ b/.changeset/beige-lamps-cry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +support advanced querying capabilities using the config option `queryMode` diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index e0ead5e6d8..d9488486ad 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -35,6 +35,11 @@ catalog: # the App registration in the Microsoft Azure Portal. clientId: ${MICROSOFT_GRAPH_CLIENT_ID} clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} + # Optional mode for querying which defaults to "basic". + # By default, the Microsoft Graph API only provides the basic feature set + # for querying. Certain features are limited to advanced querying capabilities. + # (See https://docs.microsoft.com/en-us/graph/aad-advanced-queries) + queryMode: basic # basic | advanced # Optional parameter to include the expanded resource or collection referenced # by a single relationship (navigation property) in your results. # Only one relationship can be expanded in a single request. diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index f7d0f112ad..e3840dc6f7 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -70,7 +70,10 @@ export class MicrosoftGraphClient { groupId: string, maxSize: number, ): Promise; - getGroups(query?: ODataQuery): AsyncIterable; + getGroups( + query?: ODataQuery, + queryMode?: 'basic' | 'advanced', + ): AsyncIterable; getOrganization(tenantId: string): Promise; // (undocumented) getUserPhoto(userId: string, sizeId?: string): Promise; @@ -82,13 +85,20 @@ export class MicrosoftGraphClient { userId: string, query?: ODataQuery, ): Promise; - getUsers(query?: ODataQuery): AsyncIterable; + getUsers( + query?: ODataQuery, + queryMode?: 'basic' | 'advanced', + ): AsyncIterable; requestApi( path: string, query?: ODataQuery, headers?: Record, ): Promise; - requestCollection(path: string, query?: ODataQuery): AsyncIterable; + requestCollection( + path: string, + query?: ODataQuery, + queryMode?: 'basic' | 'advanced', + ): AsyncIterable; requestRaw( url: string, headers?: Record, @@ -167,6 +177,7 @@ export type MicrosoftGraphProviderConfig = { groupExpand?: string; groupFilter?: string; groupSearch?: string; + queryMode?: 'basic' | 'advanced'; }; // @public @@ -178,6 +189,7 @@ export type ODataQuery = { filter?: string; expand?: string; select?: string[]; + count?: boolean; }; // @public @@ -202,6 +214,7 @@ export function readMicrosoftGraphOrg( groupExpand?: string; groupSearch?: string; groupFilter?: string; + queryMode?: 'basic' | 'advanced'; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts index cba5d9f640..20e791c5d5 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts @@ -24,6 +24,7 @@ import { MicrosoftGraphProviderConfig } from './config'; * OData (Open Data Protocol) Query * * {@link https://docs.microsoft.com/en-us/odata/concepts/queryoptions-overview} + * {@link https://docs.microsoft.com/en-us/graph/query-parameters} * @public */ export type ODataQuery = { @@ -43,6 +44,10 @@ export type ODataQuery = { * request a specific set of properties for each entity or complex type */ select?: string[]; + /** + * Retrieves the total count of matching resources. + */ + count?: boolean; }; /** @@ -100,19 +105,34 @@ export class MicrosoftGraphClient { * @public * @param path - Resource in Microsoft Graph * @param query - OData Query {@link ODataQuery} - * + * @param queryMode - Mode to use while querying. Some features are only available at "advanced". */ async *requestCollection( path: string, query?: ODataQuery, + queryMode?: 'basic' | 'advanced', ): AsyncIterable { - const headers: Record = query?.search - ? { - // Eventual consistency is required to use $search. - // If a new user/group is not found, it'll eventually be imported on a subsequent read - ConsistencyLevel: 'eventual', - } - : {}; + // upgrade to advanced query mode transparently when "search" is used + // to stay backwards compatible. + const appliedQueryMode = query?.search ? 'advanced' : queryMode ?? 'basic'; + + // not needed for "search" + // as of https://docs.microsoft.com/en-us/graph/aad-advanced-queries?tabs=http + // even though a few other places say the opposite + // - https://docs.microsoft.com/en-us/graph/api/user-list?view=graph-rest-1.0&tabs=http#request-headers + // - https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties + if (appliedQueryMode === 'advanced' && (query?.filter || query?.select)) { + query.count = true; + } + const headers: Record = + appliedQueryMode === 'advanced' + ? { + // Eventual consistency is required for advanced querying capabilities + // like "$search" or parts of "$filter". + // If a new user/group is not found, it'll eventually be imported on a subsequent read + ConsistencyLevel: 'eventual', + } + : {}; let response = await this.requestApi(path, query, headers); @@ -156,6 +176,7 @@ export class MicrosoftGraphClient { $filter: query?.filter, $select: query?.select?.join(','), $expand: query?.expand, + $count: query?.count, }, { addQueryPrefix: true, @@ -248,10 +269,17 @@ export class MicrosoftGraphClient { * * @public * @param query - OData Query {@link ODataQuery} - * + * @param queryMode - Mode to use while querying. Some features are only available at "advanced". */ - async *getUsers(query?: ODataQuery): AsyncIterable { - yield* this.requestCollection(`users`, query); + async *getUsers( + query?: ODataQuery, + queryMode?: 'basic' | 'advanced', + ): AsyncIterable { + yield* this.requestCollection( + `users`, + query, + queryMode, + ); } /** @@ -280,12 +308,20 @@ export class MicrosoftGraphClient { * Get a collection of * {@link https://docs.microsoft.com/en-us/graph/api/resources/group | Group} * from Graph API and return as `AsyncIterable` + * * @public * @param query - OData Query {@link ODataQuery} - * + * @param queryMode - Mode to use while querying. Some features are only available at "advanced". */ - async *getGroups(query?: ODataQuery): AsyncIterable { - yield* this.requestCollection(`groups`, query); + async *getGroups( + query?: ODataQuery, + queryMode?: 'basic' | 'advanced', + ): AsyncIterable { + yield* this.requestCollection( + `groups`, + query, + queryMode, + ); } /** diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index cfc5c1cbb0..34796fb29a 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -93,4 +93,21 @@ describe('readMicrosoftGraphConfig', () => { }; expect(() => readMicrosoftGraphConfig(new ConfigReader(config))).toThrow(); }); + + it('should fail if both userFilter and userGroupMemberSearch are set', () => { + const config = { + providers: [ + { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.example.com/', + userFilter: 'accountEnabled eq true', + userGroupMemberSearch: 'any', + }, + ], + }; + expect(() => readMicrosoftGraphConfig(new ConfigReader(config))).toThrow(); + }); }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index b7fbcfb8a9..441892531a 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -53,7 +53,7 @@ export type MicrosoftGraphProviderConfig = { */ userFilter?: string; /** - * The expand argument to apply to users. + * The "expand" argument to apply to users. * * E.g. "manager" */ @@ -88,6 +88,15 @@ export type MicrosoftGraphProviderConfig = { * E.g. "\"displayName:-team\"" would only match groups which contain '-team' */ groupSearch?: string; + /** + * By default, the Microsoft Graph API only provides the basic feature set + * for querying. Certain features are limited to advanced query capabilities + * (see https://docs.microsoft.com/en-us/graph/aad-advanced-queries) + * and need to be enabled. + * + * Some features like `$expand` are not available for advanced queries, though. + */ + queryMode?: 'basic' | 'advanced'; }; /** @@ -136,6 +145,15 @@ export function readMicrosoftGraphConfig( ); } + const queryMode = providerConfig.getOptionalString('queryMode'); + if ( + queryMode !== undefined && + queryMode !== 'basic' && + queryMode !== 'advanced' + ) { + throw new Error(`queryMode must be one of: basic, advanced`); + } + providers.push({ target, authority, @@ -149,6 +167,7 @@ export function readMicrosoftGraphConfig( groupExpand, groupFilter, groupSearch, + queryMode, }); } diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index e3d6897d66..1fc4904b74 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -111,9 +111,62 @@ describe('read microsoft graph', () => { ]); expect(client.getUsers).toBeCalledTimes(1); - expect(client.getUsers).toBeCalledWith({ - filter: 'accountEnabled eq true', + expect(client.getUsers).toBeCalledWith( + { + filter: 'accountEnabled eq true', + }, + undefined, + ); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); + }); + + it('should read users with advanced query mode', async () => { + async function* getExampleUsers() { + yield { + id: 'userid', + displayName: 'User Name', + mail: 'user.name@example.com', + }; + } + + client.getUsers.mockImplementation(getExampleUsers); + client.getUserPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { users } = await readMicrosoftGraphUsers(client, { + queryMode: 'advanced', + userFilter: 'accountEnabled eq true', + logger: getVoidLogger(), }); + + expect(users).toEqual([ + user({ + metadata: { + annotations: { + 'graph.microsoft.com/user-id': 'userid', + }, + name: 'user.name_example.com', + }, + spec: { + profile: { + displayName: 'User Name', + email: 'user.name@example.com', + picture: 'data:image/jpeg;base64,...', + }, + memberOf: [], + }, + }), + ]); + + expect(client.getUsers).toBeCalledTimes(1); + expect(client.getUsers).toBeCalledWith( + { + filter: 'accountEnabled eq true', + }, + 'advanced', + ); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); @@ -154,10 +207,13 @@ describe('read microsoft graph', () => { ]); expect(client.getUsers).toBeCalledTimes(1); - expect(client.getUsers).toBeCalledWith({ - expand: 'manager', - filter: 'accountEnabled eq true', - }); + expect(client.getUsers).toBeCalledWith( + { + expand: 'manager', + filter: 'accountEnabled eq true', + }, + undefined, + ); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); @@ -222,9 +278,88 @@ describe('read microsoft graph', () => { ]); expect(client.getGroups).toBeCalledTimes(1); - expect(client.getGroups).toBeCalledWith({ - filter: 'securityEnabled eq true', + expect(client.getGroups).toBeCalledWith( + { + filter: 'securityEnabled eq true', + }, + undefined, + ); + expect(client.getGroupMembers).toBeCalledTimes(1); + expect(client.getGroupMembers).toBeCalledWith('groupid'); + + expect(client.getUserProfile).toBeCalledTimes(1); + expect(client.getUserProfile).toBeCalledWith('userid', { + expand: undefined, }); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); + }); + + it('should read users from Groups with advanced query mode', async () => { + async function* getExampleGroups() { + yield { + id: 'groupid', + displayName: 'Group Name', + description: 'Group Description', + mail: 'group@example.com', + }; + } + + async function* getExampleGroupMembers(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.group', + id: 'childgroupid', + }; + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + + client.getUserProfile.mockResolvedValue({ + id: 'userid', + displayName: 'User Name', + mail: 'user.name@example.com', + }); + client.getUserPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { users } = await readMicrosoftGraphUsersInGroups(client, { + queryMode: 'advanced', + userGroupMemberFilter: 'securityEnabled eq true', + logger: getVoidLogger(), + }); + + expect(users).toEqual([ + user({ + metadata: { + annotations: { + 'graph.microsoft.com/user-id': 'userid', + }, + name: 'user.name_example.com', + }, + spec: { + profile: { + displayName: 'User Name', + email: 'user.name@example.com', + picture: 'data:image/jpeg;base64,...', + }, + memberOf: [], + }, + }), + ]); + + expect(client.getGroups).toBeCalledTimes(1); + expect(client.getGroups).toBeCalledWith( + { + filter: 'securityEnabled eq true', + }, + 'advanced', + ); expect(client.getGroupMembers).toBeCalledTimes(1); expect(client.getGroupMembers).toBeCalledWith('groupid'); @@ -292,10 +427,13 @@ describe('read microsoft graph', () => { ]); expect(client.getGroups).toBeCalledTimes(1); - expect(client.getGroups).toBeCalledWith({ - expand: 'member', - filter: 'securityEnabled eq true', - }); + expect(client.getGroups).toBeCalledWith( + { + expand: 'member', + filter: 'securityEnabled eq true', + }, + undefined, + ); expect(client.getGroupMembers).toBeCalledTimes(1); expect(client.getGroupMembers).toBeCalledWith('groupid'); @@ -444,9 +582,108 @@ describe('read microsoft graph', () => { expect(groupMember.get('organization_name')).toEqual(new Set()); expect(client.getGroups).toBeCalledTimes(1); - expect(client.getGroups).toBeCalledWith({ - filter: 'securityEnabled eq false', + expect(client.getGroups).toBeCalledWith( + { + filter: 'securityEnabled eq false', + }, + undefined, + ); + expect(client.getGroupMembers).toBeCalledTimes(1); + expect(client.getGroupMembers).toBeCalledWith('groupid'); + // TODO: Loading groups photos doesn't work right now as Microsoft Graph + // doesn't allows this yet + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1); + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); + }); + + it('should read groups with advanced query mode', async () => { + async function* getExampleGroups() { + yield { + id: 'groupid', + displayName: 'Group Name', + description: 'Group Description', + mail: 'group@example.com', + }; + } + + async function* getExampleGroupMembers(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.group', + id: 'childgroupid', + }; + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', }); + client.getGroupPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { groups, groupMember, groupMemberOf, rootGroup } = + await readMicrosoftGraphGroups(client, 'tenantid', { + queryMode: 'advanced', + groupFilter: 'securityEnabled eq false', + }); + + const expectedRootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + profile: { + displayName: 'Organization Name', + }, + children: [], + }, + }); + expect(groups).toEqual([ + expectedRootGroup, + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'groupid', + }, + name: 'group_name', + description: 'Group Description', + }, + spec: { + type: 'team', + profile: { + displayName: 'Group Name', + email: 'group@example.com', + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet + /* picture: 'data:image/jpeg;base64,...',*/ + }, + children: [], + }, + }), + ]); + expect(rootGroup).toEqual(expectedRootGroup); + expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid'])); + expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid'])); + expect(groupMember.get('organization_name')).toEqual(new Set()); + + expect(client.getGroups).toBeCalledTimes(1); + expect(client.getGroups).toBeCalledWith( + { + filter: 'securityEnabled eq false', + }, + 'advanced', + ); expect(client.getGroupMembers).toBeCalledTimes(1); expect(client.getGroupMembers).toBeCalledWith('groupid'); // TODO: Loading groups photos doesn't work right now as Microsoft Graph @@ -537,10 +774,13 @@ describe('read microsoft graph', () => { expect(groupMember.get('organization_name')).toEqual(new Set()); expect(client.getGroups).toBeCalledTimes(1); - expect(client.getGroups).toBeCalledWith({ - expand: 'member', - filter: 'securityEnabled eq false', - }); + expect(client.getGroups).toBeCalledWith( + { + expand: 'member', + filter: 'securityEnabled eq false', + }, + undefined, + ); expect(client.getGroupMembers).toBeCalledTimes(1); expect(client.getGroupMembers).toBeCalledWith('groupid'); // TODO: Loading groups photos doesn't work right now as Microsoft Graph @@ -628,9 +868,12 @@ describe('read microsoft graph', () => { }), ]); expect(rootGroup).toEqual(expectedRootGroup); - expect(client.getGroups).toBeCalledWith({ - filter: 'securityEnabled eq true', - }); + expect(client.getGroups).toBeCalledWith( + { + filter: 'securityEnabled eq true', + }, + undefined, + ); expect(client.getGroupMembers).toBeCalledTimes(1); expect(client.getGroupMembers).toBeCalledWith('groupid'); }); @@ -788,13 +1031,19 @@ describe('read microsoft graph', () => { }); expect(client.getUsers).toBeCalledTimes(1); - expect(client.getUsers).toBeCalledWith({ - filter: undefined, - }); + expect(client.getUsers).toBeCalledWith( + { + filter: undefined, + }, + undefined, + ); expect(client.getGroups).toBeCalledTimes(1); - expect(client.getGroups).toBeCalledWith({ - filter: 'securityEnabled eq false', - }); + expect(client.getGroups).toBeCalledWith( + { + filter: 'securityEnabled eq false', + }, + undefined, + ); }); it('should read users using userExpand and userFilter', async () => { @@ -822,14 +1071,20 @@ describe('read microsoft graph', () => { }); expect(client.getUsers).toBeCalledTimes(1); - expect(client.getUsers).toBeCalledWith({ - expand: 'manager', - filter: 'accountEnabled eq true', - }); + expect(client.getUsers).toBeCalledWith( + { + expand: 'manager', + filter: 'accountEnabled eq true', + }, + undefined, + ); expect(client.getGroups).toBeCalledTimes(1); - expect(client.getGroups).toBeCalledWith({ - filter: 'securityEnabled eq false', - }); + expect(client.getGroups).toBeCalledWith( + { + filter: 'securityEnabled eq false', + }, + undefined, + ); }); it('should read users using userExpand and userGroupMemberFilter', async () => { @@ -858,12 +1113,18 @@ describe('read microsoft graph', () => { expect(client.getUsers).toBeCalledTimes(0); expect(client.getGroups).toBeCalledTimes(2); - expect(client.getGroups).toBeCalledWith({ - filter: 'name eq backstage-group', - }); - expect(client.getGroups).toBeCalledWith({ - filter: 'securityEnabled eq false', - }); + expect(client.getGroups).toBeCalledWith( + { + filter: 'name eq backstage-group', + }, + undefined, + ); + expect(client.getGroups).toBeCalledWith( + { + filter: 'securityEnabled eq false', + }, + undefined, + ); expect(client.getUserProfile).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index ed6ca76e68..7174a6012c 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -84,6 +84,7 @@ export async function defaultUserTransformer( export async function readMicrosoftGraphUsers( client: MicrosoftGraphClient, options: { + queryMode?: 'basic' | 'advanced'; userFilter?: string; userExpand?: string; transformer?: UserTransformer; @@ -95,13 +96,16 @@ export async function readMicrosoftGraphUsers( const users: UserEntity[] = []; const limiter = limiterFactory(10); - const transformer = options?.transformer ?? defaultUserTransformer; + const transformer = options.transformer ?? defaultUserTransformer; const promises: Promise[] = []; - for await (const user of client.getUsers({ - filter: options.userFilter, - expand: options.userExpand, - })) { + for await (const user of client.getUsers( + { + filter: options.userFilter, + expand: options.userExpand, + }, + options.queryMode, + )) { // Process all users in parallel, otherwise it can take quite some time promises.push( limiter(async () => { @@ -137,6 +141,7 @@ export async function readMicrosoftGraphUsers( export async function readMicrosoftGraphUsersInGroups( client: MicrosoftGraphClient, options: { + queryMode?: 'basic' | 'advanced'; userExpand?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; @@ -157,11 +162,14 @@ export async function readMicrosoftGraphUsersInGroups( const groupMemberUsers: Set = new Set(); - for await (const group of client.getGroups({ - expand: options.groupExpand, - search: options.userGroupMemberSearch, - filter: options.userGroupMemberFilter, - })) { + for await (const group of client.getGroups( + { + expand: options.groupExpand, + search: options.userGroupMemberSearch, + filter: options.userGroupMemberFilter, + }, + options.queryMode, + )) { // Process all groups in parallel, otherwise it can take quite some time userGroupMemberPromises.push( limiter(async () => { @@ -331,6 +339,7 @@ export async function readMicrosoftGraphGroups( client: MicrosoftGraphClient, tenantId: string, options?: { + queryMode?: 'basic' | 'advanced'; groupExpand?: string; groupFilter?: string; groupSearch?: string; @@ -359,11 +368,14 @@ export async function readMicrosoftGraphGroups( const transformer = options?.groupTransformer ?? defaultGroupTransformer; const promises: Promise[] = []; - for await (const group of client.getGroups({ - expand: options?.groupExpand, - search: options?.groupSearch, - filter: options?.groupFilter, - })) { + for await (const group of client.getGroups( + { + expand: options?.groupExpand, + search: options?.groupSearch, + filter: options?.groupFilter, + }, + options?.queryMode, + )) { // Process all groups in parallel, otherwise it can take quite some time promises.push( limiter(async () => { @@ -520,6 +532,7 @@ export async function readMicrosoftGraphOrg( groupExpand?: string; groupSearch?: string; groupFilter?: string; + queryMode?: 'basic' | 'advanced'; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; @@ -528,10 +541,11 @@ export async function readMicrosoftGraphOrg( ): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> { const users: UserEntity[] = []; - if (options.userGroupMemberFilter) { + if (options.userGroupMemberFilter || options.userGroupMemberSearch) { const { users: usersInGroups } = await readMicrosoftGraphUsersInGroups( client, { + queryMode: options.queryMode, userGroupMemberFilter: options.userGroupMemberFilter, userGroupMemberSearch: options.userGroupMemberSearch, transformer: options.userTransformer, @@ -541,6 +555,7 @@ export async function readMicrosoftGraphOrg( users.push(...usersInGroups); } else { const { users: usersWithFilter } = await readMicrosoftGraphUsers(client, { + queryMode: options.queryMode, userFilter: options.userFilter, userExpand: options.userExpand, transformer: options.userTransformer, @@ -550,10 +565,11 @@ export async function readMicrosoftGraphOrg( } const { groups, rootGroup, groupMember, groupMemberOf } = await readMicrosoftGraphGroups(client, tenantId, { - groupSearch: options?.groupSearch, - groupFilter: options?.groupFilter, - groupTransformer: options?.groupTransformer, - organizationTransformer: options?.organizationTransformer, + queryMode: options.queryMode, + groupSearch: options.groupSearch, + groupFilter: options.groupFilter, + groupTransformer: options.groupTransformer, + organizationTransformer: options.organizationTransformer, }); resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf); diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 7d27e87d73..71aabaeef2 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -126,6 +126,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { userGroupMemberSearch: provider.userGroupMemberSearch, groupFilter: provider.groupFilter, groupSearch: provider.groupSearch, + queryMode: provider.queryMode, groupTransformer: this.options.groupTransformer, userTransformer: this.options.userTransformer, organizationTransformer: this.options.organizationTransformer, diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 9037bd8728..5d8619e874 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -73,6 +73,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { this.groupTransformer = options.groupTransformer; this.organizationTransformer = options.organizationTransformer; } + getProcessorName(): string { return 'MicrosoftGraphOrgReaderProcessor'; } @@ -95,7 +96,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { ); } - // Read out all of the raw data + // Read out all the raw data const startTimestamp = Date.now(); this.logger.info('Reading Microsoft Graph users and groups'); @@ -112,6 +113,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { groupExpand: provider.groupExpand, groupFilter: provider.groupFilter, groupSearch: provider.groupSearch, + queryMode: provider.queryMode, userTransformer: this.userTransformer, groupTransformer: this.groupTransformer, organizationTransformer: this.organizationTransformer,