From da7651605a6c78e8e4a89ab89fc2d0e30291d06b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20St=C3=B6rkle?= Date: Fri, 16 Feb 2024 13:15:50 +0100 Subject: [PATCH] feat(catalog): ingest member groups if configured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Störkle --- .../config.d.ts | 5 + .../src/microsoftGraph/config.test.ts | 2 + .../src/microsoftGraph/config.ts | 10 ++ .../src/microsoftGraph/read.test.ts | 114 ++++++++++++++++++ .../src/microsoftGraph/read.ts | 28 +++++ .../MicrosoftGraphOrgEntityProvider.ts | 1 + 6 files changed, 160 insertions(+) diff --git a/plugins/catalog-backend-module-msgraph/config.d.ts b/plugins/catalog-backend-module-msgraph/config.d.ts index 5a1f56a03f..a470c075f0 100644 --- a/plugins/catalog-backend-module-msgraph/config.d.ts +++ b/plugins/catalog-backend-module-msgraph/config.d.ts @@ -192,6 +192,11 @@ export interface Config { * E.g. ["id", "displayName", "description"] */ select?: string[]; + /** + * Whether to ingest groups that are members of the found/filtered/searched groups. + * Default value is `false`. + */ + includeSubGroups?: boolean; }; userGroupMember?: { diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index b02894dfd9..e1c81662a3 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -175,6 +175,7 @@ describe('readProviderConfigs', () => { expand: 'member', filter: 'securityEnabled eq false', select: ['id', 'displayName', 'description'], + includeSubGroups: true, }, schedule: { frequency: 'PT30M', @@ -203,6 +204,7 @@ describe('readProviderConfigs', () => { groupExpand: 'member', groupSelect: ['id', 'displayName', 'description'], groupFilter: 'securityEnabled eq false', + groupIncludeSubGroups: true, schedule: { frequency: Duration.fromISO('PT30M'), timeout: { diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index 1545114e45..257a7d5736 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -116,6 +116,12 @@ export type MicrosoftGraphProviderConfig = { */ groupSelect?: string[]; + /** + * Whether to ingest groups that are members of the found/filtered/searched groups. + * Default value is `false`. + */ + groupIncludeSubGroups?: boolean; + /** * By default, the Microsoft Graph API only provides the basic feature set * for querying. Certain features are limited to advanced query capabilities @@ -292,6 +298,9 @@ export function readProviderConfig( const groupFilter = config.getOptionalString('group.filter'); const groupSearch = config.getOptionalString('group.search'); const groupSelect = config.getOptionalStringArray('group.select'); + const groupIncludeSubGroups = config.getOptionalBoolean( + 'group.includeSubGroups', + ); const queryMode = config.getOptionalString('queryMode'); if ( @@ -347,6 +356,7 @@ export function readProviderConfig( groupFilter, groupSearch, groupSelect, + groupIncludeSubGroups, queryMode, userGroupMemberFilter, userGroupMemberSearch, diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index 8821ffd2db..b83597653f 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -90,6 +90,9 @@ describe('read microsoft graph', () => { yield { '@odata.type': '#microsoft.graph.group', id: 'childgroupid', + displayName: 'Child Group Name', + description: 'Child Group Description', + mail: 'childgroup@example.com', }; yield { '@odata.type': '#microsoft.graph.user', @@ -770,6 +773,117 @@ describe('read microsoft graph', () => { top: 999, }); }); + + it('should read groups and their sub groups', async () => { + async function* getExampleGroupMembersForSubGroup(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid2', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementationOnce(getExampleGroupMembers); + client.getGroupMembers.mockImplementationOnce( + getExampleGroupMembersForSubGroup, + ); + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + client.getGroupPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { groups, groupMember, groupMemberOf, rootGroup } = + await readMicrosoftGraphGroups(client, 'tenantid', { + groupIncludeSubGroups: true, + }); + + const expectedRootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + profile: { + displayName: 'Organization Name', + }, + children: [], + }, + }); + expect(groups).toEqual([ + expectedRootGroup, + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'childgroupid', + }, + name: 'child_group_name', + description: 'Child Group Description', + }, + spec: { + type: 'team', + profile: { + displayName: 'Child Group Name', + email: 'childgroup@example.com', + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet + /* picture: 'data:image/jpeg;base64,...',*/ + }, + children: [], + }, + }), + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'groupid', + }, + name: 'group_name', + description: 'Group Description', + }, + spec: { + type: 'team', + profile: { + displayName: 'Group Name', + email: 'group@example.com', + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet + /* picture: 'data:image/jpeg;base64,...',*/ + }, + children: [], + }, + }), + ]); + expect(rootGroup).toEqual(expectedRootGroup); + expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid'])); + expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid'])); + expect(groupMemberOf.get('userid2')).toEqual(new Set(['childgroupid'])); + expect(groupMember.get('organization_name')).toEqual(new Set()); + + expect(client.getGroups).toHaveBeenCalledTimes(1); + expect(client.getGroups).toHaveBeenCalledWith( + { + top: 999, + }, + undefined, + ); + expect(client.getGroupMembers).toHaveBeenCalledTimes(2); + expect(client.getGroupMembers).toHaveBeenCalledWith('groupid', { + top: 999, + }); + expect(client.getGroupMembers).toHaveBeenCalledWith('childgroupid', { + top: 999, + }); + // TODO: Loading groups photos doesn't work right now as Microsoft Graph + // doesn't allows this yet + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1); + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); + }); }); describe('resolveRelations', () => { diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index 2ca21fca9c..60e5c3d79b 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -178,6 +178,7 @@ export async function readMicrosoftGraphGroups( groupFilter?: string; groupSearch?: string; groupSelect?: string[]; + groupIncludeSubGroups?: boolean; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; }, @@ -244,6 +245,31 @@ export async function readMicrosoftGraphGroups( if (member['@odata.type'] === '#microsoft.graph.group') { ensureItem(groupMember, group.id!, member.id); + + if (options?.groupIncludeSubGroups) { + const groupMemberEntity = await transformer(member); + + if (groupMemberEntity) { + groups.push(groupMemberEntity); + + for await (const subMember of client.getGroupMembers( + member.id!, + { top: PAGE_SIZE }, + )) { + if (!subMember.id) { + continue; + } + + if (subMember['@odata.type'] === '#microsoft.graph.user') { + ensureItem(groupMemberOf, subMember.id, member.id!); + } + + if (subMember['@odata.type'] === '#microsoft.graph.group') { + ensureItem(groupMember, member.id!, subMember.id); + } + } + } + } } } @@ -377,6 +403,7 @@ export async function readMicrosoftGraphOrg( groupSearch?: string; groupFilter?: string; groupSelect?: string[]; + groupIncludeSubGroups?: boolean; queryMode?: 'basic' | 'advanced'; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; @@ -421,6 +448,7 @@ export async function readMicrosoftGraphOrg( groupFilter: options.groupFilter, groupSearch: options.groupSearch, groupSelect: options.groupSelect, + groupIncludeSubGroups: options.groupIncludeSubGroups, groupTransformer: options.groupTransformer, organizationTransformer: options.organizationTransformer, }); diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 3c8b903e69..c48ee18ffe 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -317,6 +317,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { groupFilter: provider.groupFilter, groupSearch: provider.groupSearch, groupSelect: provider.groupSelect, + groupIncludeSubGroups: provider.groupIncludeSubGroups, queryMode: provider.queryMode, groupTransformer: this.options.groupTransformer, userTransformer: this.options.userTransformer,