From a0fd89c3ef77677404cfb46e973000ef4061d2b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20St=C3=B6rkle?= Date: Thu, 15 Feb 2024 14:54:48 +0100 Subject: [PATCH 1/5] fix: use correct odata type in TypeScript MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Störkle --- .../src/microsoftGraph/client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts index 1233b1e2f9..c190e2a959 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts @@ -64,8 +64,8 @@ export type ODataQuery = { * @public */ export type GroupMember = - | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.user' }) - | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.group' }); + | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.group' }) + | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.user' }); /** * A HTTP Client that communicates with Microsoft Graph API. From da7651605a6c78e8e4a89ab89fc2d0e30291d06b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20St=C3=B6rkle?= Date: Fri, 16 Feb 2024 13:15:50 +0100 Subject: [PATCH 2/5] feat(catalog): ingest member groups if configured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Störkle --- .../config.d.ts | 5 + .../src/microsoftGraph/config.test.ts | 2 + .../src/microsoftGraph/config.ts | 10 ++ .../src/microsoftGraph/read.test.ts | 114 ++++++++++++++++++ .../src/microsoftGraph/read.ts | 28 +++++ .../MicrosoftGraphOrgEntityProvider.ts | 1 + 6 files changed, 160 insertions(+) diff --git a/plugins/catalog-backend-module-msgraph/config.d.ts b/plugins/catalog-backend-module-msgraph/config.d.ts index 5a1f56a03f..a470c075f0 100644 --- a/plugins/catalog-backend-module-msgraph/config.d.ts +++ b/plugins/catalog-backend-module-msgraph/config.d.ts @@ -192,6 +192,11 @@ export interface Config { * E.g. ["id", "displayName", "description"] */ select?: string[]; + /** + * Whether to ingest groups that are members of the found/filtered/searched groups. + * Default value is `false`. + */ + includeSubGroups?: boolean; }; userGroupMember?: { diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index b02894dfd9..e1c81662a3 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -175,6 +175,7 @@ describe('readProviderConfigs', () => { expand: 'member', filter: 'securityEnabled eq false', select: ['id', 'displayName', 'description'], + includeSubGroups: true, }, schedule: { frequency: 'PT30M', @@ -203,6 +204,7 @@ describe('readProviderConfigs', () => { groupExpand: 'member', groupSelect: ['id', 'displayName', 'description'], groupFilter: 'securityEnabled eq false', + groupIncludeSubGroups: true, schedule: { frequency: Duration.fromISO('PT30M'), timeout: { diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index 1545114e45..257a7d5736 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -116,6 +116,12 @@ export type MicrosoftGraphProviderConfig = { */ groupSelect?: string[]; + /** + * Whether to ingest groups that are members of the found/filtered/searched groups. + * Default value is `false`. + */ + groupIncludeSubGroups?: boolean; + /** * By default, the Microsoft Graph API only provides the basic feature set * for querying. Certain features are limited to advanced query capabilities @@ -292,6 +298,9 @@ export function readProviderConfig( const groupFilter = config.getOptionalString('group.filter'); const groupSearch = config.getOptionalString('group.search'); const groupSelect = config.getOptionalStringArray('group.select'); + const groupIncludeSubGroups = config.getOptionalBoolean( + 'group.includeSubGroups', + ); const queryMode = config.getOptionalString('queryMode'); if ( @@ -347,6 +356,7 @@ export function readProviderConfig( groupFilter, groupSearch, groupSelect, + groupIncludeSubGroups, queryMode, userGroupMemberFilter, userGroupMemberSearch, diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index 8821ffd2db..b83597653f 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -90,6 +90,9 @@ describe('read microsoft graph', () => { yield { '@odata.type': '#microsoft.graph.group', id: 'childgroupid', + displayName: 'Child Group Name', + description: 'Child Group Description', + mail: 'childgroup@example.com', }; yield { '@odata.type': '#microsoft.graph.user', @@ -770,6 +773,117 @@ describe('read microsoft graph', () => { top: 999, }); }); + + it('should read groups and their sub groups', async () => { + async function* getExampleGroupMembersForSubGroup(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid2', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementationOnce(getExampleGroupMembers); + client.getGroupMembers.mockImplementationOnce( + getExampleGroupMembersForSubGroup, + ); + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + client.getGroupPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { groups, groupMember, groupMemberOf, rootGroup } = + await readMicrosoftGraphGroups(client, 'tenantid', { + groupIncludeSubGroups: true, + }); + + const expectedRootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + profile: { + displayName: 'Organization Name', + }, + children: [], + }, + }); + expect(groups).toEqual([ + expectedRootGroup, + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'childgroupid', + }, + name: 'child_group_name', + description: 'Child Group Description', + }, + spec: { + type: 'team', + profile: { + displayName: 'Child Group Name', + email: 'childgroup@example.com', + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet + /* picture: 'data:image/jpeg;base64,...',*/ + }, + children: [], + }, + }), + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'groupid', + }, + name: 'group_name', + description: 'Group Description', + }, + spec: { + type: 'team', + profile: { + displayName: 'Group Name', + email: 'group@example.com', + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet + /* picture: 'data:image/jpeg;base64,...',*/ + }, + children: [], + }, + }), + ]); + expect(rootGroup).toEqual(expectedRootGroup); + expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid'])); + expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid'])); + expect(groupMemberOf.get('userid2')).toEqual(new Set(['childgroupid'])); + expect(groupMember.get('organization_name')).toEqual(new Set()); + + expect(client.getGroups).toHaveBeenCalledTimes(1); + expect(client.getGroups).toHaveBeenCalledWith( + { + top: 999, + }, + undefined, + ); + expect(client.getGroupMembers).toHaveBeenCalledTimes(2); + expect(client.getGroupMembers).toHaveBeenCalledWith('groupid', { + top: 999, + }); + expect(client.getGroupMembers).toHaveBeenCalledWith('childgroupid', { + top: 999, + }); + // TODO: Loading groups photos doesn't work right now as Microsoft Graph + // doesn't allows this yet + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1); + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); + }); }); describe('resolveRelations', () => { diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index 2ca21fca9c..60e5c3d79b 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -178,6 +178,7 @@ export async function readMicrosoftGraphGroups( groupFilter?: string; groupSearch?: string; groupSelect?: string[]; + groupIncludeSubGroups?: boolean; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; }, @@ -244,6 +245,31 @@ export async function readMicrosoftGraphGroups( if (member['@odata.type'] === '#microsoft.graph.group') { ensureItem(groupMember, group.id!, member.id); + + if (options?.groupIncludeSubGroups) { + const groupMemberEntity = await transformer(member); + + if (groupMemberEntity) { + groups.push(groupMemberEntity); + + for await (const subMember of client.getGroupMembers( + member.id!, + { top: PAGE_SIZE }, + )) { + if (!subMember.id) { + continue; + } + + if (subMember['@odata.type'] === '#microsoft.graph.user') { + ensureItem(groupMemberOf, subMember.id, member.id!); + } + + if (subMember['@odata.type'] === '#microsoft.graph.group') { + ensureItem(groupMember, member.id!, subMember.id); + } + } + } + } } } @@ -377,6 +403,7 @@ export async function readMicrosoftGraphOrg( groupSearch?: string; groupFilter?: string; groupSelect?: string[]; + groupIncludeSubGroups?: boolean; queryMode?: 'basic' | 'advanced'; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; @@ -421,6 +448,7 @@ export async function readMicrosoftGraphOrg( groupFilter: options.groupFilter, groupSearch: options.groupSearch, groupSelect: options.groupSelect, + groupIncludeSubGroups: options.groupIncludeSubGroups, groupTransformer: options.groupTransformer, organizationTransformer: options.organizationTransformer, }); diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 3c8b903e69..c48ee18ffe 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -317,6 +317,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { groupFilter: provider.groupFilter, groupSearch: provider.groupSearch, groupSelect: provider.groupSelect, + groupIncludeSubGroups: provider.groupIncludeSubGroups, queryMode: provider.queryMode, groupTransformer: this.options.groupTransformer, userTransformer: this.options.userTransformer, From 24b3a3141b2eec8f89e4f76aa3cfc7328d28ea6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20St=C3=B6rkle?= Date: Mon, 19 Feb 2024 08:35:18 +0100 Subject: [PATCH 3/5] docs: document `includeSubGroups` config option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Störkle --- docs/integrations/azure/org.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md index 89abd4813b..b1719a63da 100644 --- a/docs/integrations/azure/org.md +++ b/docs/integrations/azure/org.md @@ -112,6 +112,17 @@ microsoftGraphOrg: search: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' ``` +If you don't want to only ingest groups matching the `search` and/or `filter` query, but also the groups which are members of the matched groups, you can use the `includeSubGroups` configuration: + +```yaml +microsoftGraphOrg: + providerId: + group: + filter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') + search: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' + includeSubGroups: true +``` + In addition to these groups, one additional group will be created for your organization. All imported groups will be a child of this group. From d9cf697afda7543d72610e59f656ca05e2c9c3d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20St=C3=B6rkle?= Date: Mon, 19 Feb 2024 08:40:38 +0100 Subject: [PATCH 4/5] chore: generate API report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Störkle --- plugins/catalog-backend-module-msgraph/api-report.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 86170e2910..87402eedb1 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -39,10 +39,10 @@ export function defaultUserTransformer( // @public export type GroupMember = | (MicrosoftGraph.Group & { - '@odata.type': '#microsoft.graph.user'; + '@odata.type': '#microsoft.graph.group'; }) | (MicrosoftGraph.User & { - '@odata.type': '#microsoft.graph.group'; + '@odata.type': '#microsoft.graph.user'; }); // @public @@ -210,6 +210,7 @@ export type MicrosoftGraphProviderConfig = { groupFilter?: string; groupSearch?: string; groupSelect?: string[]; + groupIncludeSubGroups?: boolean; queryMode?: 'basic' | 'advanced'; loadUserPhotos?: boolean; schedule?: TaskScheduleDefinition; @@ -253,6 +254,7 @@ export function readMicrosoftGraphOrg( groupSearch?: string; groupFilter?: string; groupSelect?: string[]; + groupIncludeSubGroups?: boolean; queryMode?: 'basic' | 'advanced'; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; From 58dff4d5e574428090ec09a4fbc7de3e08224c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20St=C3=B6rkle?= Date: Tue, 16 Apr 2024 10:01:13 +0200 Subject: [PATCH 5/5] chore: add changeset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Störkle --- .changeset/little-suns-fly.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/little-suns-fly.md diff --git a/.changeset/little-suns-fly.md b/.changeset/little-suns-fly.md new file mode 100644 index 0000000000..887df4be6d --- /dev/null +++ b/.changeset/little-suns-fly.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Added option to ingest groups based on their group membership in Azure Entra ID