paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 36 Packages Projects Releases Wiki Activity

Merge pull request #7084 from msamad/msamad/add-user-group-filter-to-msgraph

Browse Source 
Allow loading users using group membership
This commit is contained in:
Ben Lambert
2021-10-05 12:02:26 +02:00
committed by GitHub
parent 164d42754e 9e2358a888
commit 2ba532cb78
8 changed files with 411 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend-module-msgraph/README.md
+8
View File
@@ -55,12 +55,20 @@ catalog:
# Optional filter for user, see Microsoft Graph API for the syntax
# See https://docs.microsoft.com/en-us/graph/api/resources/user?view=graph-rest-1.0#properties
# and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter
# This and userGroupMemberFilter are mutually exclusive, only one can be specified
userFilter: accountEnabled eq true and userType eq 'member'
# Optional filter for users, use group membership to get users.
# This and userFilter are mutually exclusive, only one can be specified
userGroupMemberFilter: "displayName eq 'Backstage Users'"
# Optional filter for group, see Microsoft Graph API for the syntax
# See https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties
groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified')
```
`userFilter` and `userGroupMemberFilter` are mutually exclusive, only one can be provided. If both are provided, an error will be thrown.
By default, all users are loaded. If you want to filter users based on their attributes, use `userFilter`. `userGroupMemberFilter` can be used if you want to load users based on their group membership.
5. Add a location that ingests from Microsoft Graph:
```yaml
plugins/catalog-backend-module-msgraph/api-report.md
+2
View File
@@ -143,6 +143,7 @@ export type MicrosoftGraphProviderConfig = {
clientId: string;
clientSecret: string;
userFilter?: string;
userGroupMemberFilter?: string;
groupFilter?: string;
};
@@ -173,6 +174,7 @@ export function readMicrosoftGraphOrg(
tenantId: string,
options: {
userFilter?: string;
userGroupMemberFilter?: string;
groupFilter?: string;
userTransformer?: UserTransformer;
groupTransformer?: GroupTransformer;
plugins/catalog-backend-module-msgraph/config.d.ts
Vendored
+6
View File
@@ -73,6 +73,12 @@ export interface Config {
* E.g. "securityEnabled eq false and mailEnabled eq true"
*/
groupFilter?: string;
/**
* The filter to apply to extract users by groups memberships.
*
* E.g. "displayName eq 'Backstage Users'"
*/
userGroupMemberFilter?: string;
}>;
};
};
plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts
+10
View File
@@ -51,6 +51,12 @@ export type MicrosoftGraphProviderConfig = {
* E.g. "accountEnabled eq true and userType eq 'member'"
*/
userFilter?: string;
/**
* The filter to apply to extract users by groups memberships.
*
* E.g. "displayName eq 'Backstage Users'"
*/
userGroupMemberFilter?: string;
/**
* The filter to apply to extract groups.
*
@@ -74,6 +80,9 @@ export function readMicrosoftGraphConfig(
const clientId = providerConfig.getString('clientId');
const clientSecret = providerConfig.getString('clientSecret');
const userFilter = providerConfig.getOptionalString('userFilter');
const userGroupMemberFilter = providerConfig.getOptionalString(
'userGroupMemberFilter',
);
const groupFilter = providerConfig.getOptionalString('groupFilter');
providers.push({
@@ -83,6 +92,7 @@ export function readMicrosoftGraphConfig(
clientId,
clientSecret,
userFilter,
userGroupMemberFilter,
groupFilter,
});
}
plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts
+269
View File
@@ -19,8 +19,10 @@ import merge from 'lodash/merge';
import { GroupMember, MicrosoftGraphClient } from './client';
import {
readMicrosoftGraphGroups,
readMicrosoftGraphOrg,
readMicrosoftGraphOrganization,
readMicrosoftGraphUsers,
readMicrosoftGraphUsersInGroups,
resolveRelations,
} from './read';
import { getVoidLogger } from '@backstage/backend-common';
@@ -59,6 +61,7 @@ function group(data: Partial<GroupEntity>): GroupEntity {
describe('read microsoft graph', () => {
const client: jest.Mocked<MicrosoftGraphClient> = {
getUsers: jest.fn(),
getUserProfile: jest.fn(),
getGroups: jest.fn(),
getGroupMembers: jest.fn(),
getUserPhotoWithSizeLimit: jest.fn(),
@@ -158,6 +161,144 @@ describe('read microsoft graph', () => {
});
});
describe('readMicrosoftGraphUsersInGroups', () => {
it('should read users from Groups', async () => {
async function* getExampleGroups() {
yield {
id: 'groupid',
displayName: 'Group Name',
description: 'Group Description',
mail: 'group@example.com',
};
}
async function* getExampleGroupMembers(): AsyncIterable<GroupMember> {
yield {
'@odata.type': '#microsoft.graph.group',
id: 'childgroupid',
};
yield {
'@odata.type': '#microsoft.graph.user',
id: 'userid',
};
}
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getUserProfile.mockResolvedValue({
id: 'userid',
displayName: 'User Name',
mail: 'user.name@example.com',
});
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
const { users } = await readMicrosoftGraphUsersInGroups(client, {
userGroupMemberFilter: 'securityEnabled eq true',
logger: getVoidLogger(),
});
expect(users).toEqual([
user({
metadata: {
annotations: {
'graph.microsoft.com/user-id': 'userid',
},
name: 'user.name_example.com',
},
spec: {
profile: {
displayName: 'User Name',
email: 'user.name@example.com',
picture: 'data:image/jpeg;base64,...',
},
memberOf: [],
},
}),
]);
expect(client.getGroups).toBeCalledTimes(1);
expect(client.getGroups).toBeCalledWith({
filter: 'securityEnabled eq true',
});
expect(client.getGroupMembers).toBeCalledTimes(1);
expect(client.getGroupMembers).toBeCalledWith('groupid');
expect(client.getUserProfile).toBeCalledTimes(1);
expect(client.getUserProfile).toBeCalledWith('userid');
expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1);
expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120);
});
it('should read users with custom transformer', async () => {
async function* getExampleGroups() {
yield {
id: 'groupid',
displayName: 'Group Name',
description: 'Group Description',
mail: 'group@example.com',
};
}
async function* getExampleGroupMembers(): AsyncIterable<GroupMember> {
yield {
'@odata.type': '#microsoft.graph.group',
id: 'childgroupid',
};
yield {
'@odata.type': '#microsoft.graph.user',
id: 'userid',
};
}
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getUserProfile.mockResolvedValue({
id: 'userid',
displayName: 'User Name',
mail: 'user.name@example.com',
});
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
const { users } = await readMicrosoftGraphUsersInGroups(client, {
userGroupMemberFilter: 'securityEnabled eq true',
transformer: async () => ({
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: { name: 'x' },
spec: { memberOf: [] },
}),
logger: getVoidLogger(),
});
expect(users).toEqual([
{
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: { name: 'x' },
spec: { memberOf: [] },
},
]);
expect(client.getGroups).toBeCalledTimes(1);
expect(client.getGroups).toBeCalledWith({
filter: 'securityEnabled eq true',
});
expect(client.getGroupMembers).toBeCalledTimes(1);
expect(client.getGroupMembers).toBeCalledWith('groupid');
expect(client.getUserProfile).toBeCalledTimes(1);
expect(client.getUserProfile).toBeCalledWith('userid');
expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1);
expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120);
});
});
describe('readMicrosoftGraphOrganization', () => {
it('should read organization', async () => {
client.getOrganization.mockResolvedValue({
@@ -397,4 +538,132 @@ describe('read microsoft graph', () => {
);
});
});
describe('readMicrosoftGraphOrg', () => {
async function* getExampleUsers() {
yield {
id: 'userid',
displayName: 'User Name',
mail: 'user.name@example.com',
};
}
async function* getExampleGroups() {
yield {
id: 'groupid',
displayName: 'Group Name',
description: 'Group Description',
mail: 'group@example.com',
};
}
async function* getExampleGroupMembers(): AsyncIterable<GroupMember> {
yield {
'@odata.type': '#microsoft.graph.group',
id: 'childgroupid',
};
yield {
'@odata.type': '#microsoft.graph.user',
id: 'userid',
};
}
it('should read all users if no filter provided', async () => {
client.getOrganization.mockResolvedValue({
id: 'tenantid',
displayName: 'Organization Name',
});
client.getUsers.mockImplementation(getExampleUsers);
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getGroupPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
await readMicrosoftGraphOrg(client, 'tenantid', {
logger: getVoidLogger(),
groupFilter: 'securityEnabled eq false',
});
expect(client.getUsers).toBeCalledTimes(1);
expect(client.getUsers).toBeCalledWith({
filter: undefined,
});
expect(client.getGroups).toBeCalledTimes(1);
expect(client.getGroups).toBeCalledWith({
filter: 'securityEnabled eq false',
});
});
it('should read users using userFilter', async () => {
client.getOrganization.mockResolvedValue({
id: 'tenantid',
displayName: 'Organization Name',
});
client.getUsers.mockImplementation(getExampleUsers);
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getGroupPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
await readMicrosoftGraphOrg(client, 'tenantid', {
logger: getVoidLogger(),
userFilter: 'accountEnabled eq true',
groupFilter: 'securityEnabled eq false',
});
expect(client.getUsers).toBeCalledTimes(1);
expect(client.getUsers).toBeCalledWith({
filter: 'accountEnabled eq true',
});
expect(client.getGroups).toBeCalledTimes(1);
expect(client.getGroups).toBeCalledWith({
filter: 'securityEnabled eq false',
});
});
it('should read users using userGroupMemberFilter', async () => {
client.getOrganization.mockResolvedValue({
id: 'tenantid',
displayName: 'Organization Name',
});
client.getUsers.mockImplementation(getExampleUsers);
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getGroupPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
await readMicrosoftGraphOrg(client, 'tenantid', {
logger: getVoidLogger(),
userGroupMemberFilter: 'name eq backstage-group',
groupFilter: 'securityEnabled eq false',
});
expect(client.getUsers).toBeCalledTimes(0);
expect(client.getGroups).toBeCalledTimes(2);
expect(client.getGroups).toBeCalledWith({
filter: 'name eq backstage-group',
});
expect(client.getGroups).toBeCalledWith({
filter: 'securityEnabled eq false',
});
});
});
});
plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts
+104 -5
View File
@@ -125,6 +125,89 @@ export async function readMicrosoftGraphUsers(
return { users };
}
export async function readMicrosoftGraphUsersInGroups(
client: MicrosoftGraphClient,
options: {
userGroupMemberFilter?: string;
transformer?: UserTransformer;
logger: Logger;
},
): Promise<{
users: UserEntity[]; // With all relations empty
}> {
const users: UserEntity[] = [];
const limiter = limiterFactory(10);
const transformer = options?.transformer ?? defaultUserTransformer;
const userGroupMemberPromises: Promise<void>[] = [];
const userPromises: Promise<void>[] = [];
const groupMemberUsers: Set<string> = new Set();
for await (const group of client.getGroups({
filter: options?.userGroupMemberFilter,
})) {
// Process all groups in parallel, otherwise it can take quite some time
userGroupMemberPromises.push(
limiter(async () => {
for await (const member of client.getGroupMembers(group.id!)) {
if (!member.id) {
continue;
}
if (member['@odata.type'] === '#microsoft.graph.user') {
groupMemberUsers.add(member.id);
}
}
}),
);
}
// Wait for all group members
await Promise.all(userGroupMemberPromises);
options.logger.info(`groupMemberUsers ${groupMemberUsers.size}`);
for (const userId of groupMemberUsers) {
// Process all users in parallel, otherwise it can take quite some time
userPromises.push(
limiter(async () => {
let user;
let userPhoto;
try {
user = await client.getUserProfile(userId);
} catch (e) {
options.logger.warn(`Unable to load user for ${userId}`);
}
if (user) {
try {
userPhoto = await client.getUserPhotoWithSizeLimit(
user.id!,
// We are limiting the photo size, as users with full resolution photos
// can make the Backstage API slow
120,
);
} catch (e) {
options.logger.warn(`Unable to load userphoto for ${userId}`);
}
const entity = await transformer(user, userPhoto);
if (!entity) {
return;
}
users.push(entity);
}
}),
);
}
// Wait for all users and photos to be downloaded
await Promise.all(userPromises);
return { users };
}
export async function defaultOrganizationTransformer(
organization: MicrosoftGraph.Organization,
): Promise<GroupEntity | undefined> {
@@ -387,6 +470,7 @@ export async function readMicrosoftGraphOrg(
tenantId: string,
options: {
userFilter?: string;
userGroupMemberFilter?: string;
groupFilter?: string;
userTransformer?: UserTransformer;
groupTransformer?: GroupTransformer;
@@ -394,11 +478,26 @@ export async function readMicrosoftGraphOrg(
logger: Logger;
},
): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> {
const { users } = await readMicrosoftGraphUsers(client, {
userFilter: options.userFilter,
transformer: options.userTransformer,
logger: options.logger,
});
const users: UserEntity[] = [];
if (options.userGroupMemberFilter) {
const { users: usersInGroups } = await readMicrosoftGraphUsersInGroups(
client,
{
userGroupMemberFilter: options.userGroupMemberFilter,
transformer: options.userTransformer,
logger: options.logger,
},
);
users.push(...usersInGroups);
} else {
const { users: usersWithFilter } = await readMicrosoftGraphUsers(client, {
userFilter: options.userFilter,
transformer: options.userTransformer,
logger: options.logger,
});
users.push(...usersWithFilter);
}
const { groups, rootGroup, groupMember, groupMemberOf } =
await readMicrosoftGraphGroups(client, tenantId, {
groupFilter: options?.groupFilter,
plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts
+7
View File
@@ -90,6 +90,12 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
);
}
if (provider.userFilter && provider.userGroupMemberFilter) {
throw new Error(
`userFilter and userGroupMemberFilter are mutually exclusive, only one can be specified.`,
);
}
// Read out all of the raw data
const startTimestamp = Date.now();
this.logger.info('Reading Microsoft Graph users and groups');
@@ -101,6 +107,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
provider.tenantId,
{
userFilter: provider.userFilter,
userGroupMemberFilter: provider.userGroupMemberFilter,
groupFilter: provider.groupFilter,
userTransformer: this.userTransformer,
groupTransformer: this.groupTransformer,