paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 51 Packages Projects Releases Wiki Activity

Optimised loading of users via group membership

Browse Source 
Signed-off-by: Alex Crome <afscrome@users.noreply.github.com>
This commit is contained in:
Alex Crome
2023-01-22 23:23:26 +00:00
parent c5b119ad9c
commit fb568e2683
6 changed files with 185 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/slow-carrots-sniff.md
+8
View File
@@ -0,0 +1,8 @@
---
'@backstage/plugin-catalog-backend-module-msgraph': minor
---
Improve performance when loading users via group membership.
Users data is now loaded from a paged query, rather than having to make an extra call per user to load each user's profiles.
Note, there are still additional per user calls made to load user avatars
plugins/catalog-backend-module-msgraph/api-report.md
+11 -5
View File
@@ -70,7 +70,11 @@ export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = 'graph.microsoft.com/user-id';
export class MicrosoftGraphClient {
constructor(baseUrl: string, tokenCredential: TokenCredential);
static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient;
getGroupMembers(groupId: string): AsyncIterable<GroupMember>;
getGroupMembers(
groupId: string,
query?: ODataQuery,
queryMode?: 'basic' | 'advanced',
): AsyncIterable<GroupMember>;
// (undocumented)
getGroupPhoto(groupId: string, sizeId?: string): Promise<string | undefined>;
getGroupPhotoWithSizeLimit(
@@ -81,6 +85,11 @@ export class MicrosoftGraphClient {
query?: ODataQuery,
queryMode?: 'basic' | 'advanced',
): AsyncIterable<MicrosoftGraph.Group>;
getGroupUserMembers(
groupId: string,
query?: ODataQuery,
queryMode?: 'basic' | 'advanced',
): AsyncIterable<MicrosoftGraph.User>;
getOrganization(tenantId: string): Promise<MicrosoftGraph.Organization>;
// (undocumented)
getUserPhoto(userId: string, sizeId?: string): Promise<string | undefined>;
@@ -88,10 +97,6 @@ export class MicrosoftGraphClient {
userId: string,
maxSize: number,
): Promise<string | undefined>;
getUserProfile(
userId: string,
query?: ODataQuery,
): Promise<MicrosoftGraph.User>;
getUsers(
query?: ODataQuery,
queryMode?: 'basic' | 'advanced',
@@ -231,6 +236,7 @@ export type ODataQuery = {
expand?: string;
select?: string[];
count?: boolean;
top?: number;
};
// @public
plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts
+21 -27
View File
@@ -135,33 +135,6 @@ describe('MicrosoftGraphClient', () => {
expect(values).toEqual(['first', 'second']);
});
it('should load user profile', async () => {
worker.use(
rest.get('https://example.com/users/user-id', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
surname: 'Example',
}),
),
),
);
const userProfile = await client.getUserProfile('user-id');
expect(userProfile).toEqual({ surname: 'Example' });
});
it('should throw exception if load user profile fails', async () => {
worker.use(
rest.get('https://example.com/users/user-id', (_, res, ctx) =>
res(ctx.status(404)),
),
);
await expect(() => client.getUserProfile('user-id')).rejects.toThrow();
});
it('should load user profile photo with max size of 120', async () => {
worker.use(
rest.get('https://example.com/users/user-id/photos', (_, res, ctx) =>
@@ -330,6 +303,27 @@ describe('MicrosoftGraphClient', () => {
]);
});
it('should load user group members', async () => {
worker.use(
rest.get(
'https://example.com/groups/group-id/members/microsoft.graph.user',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
value: [{ id: '12345' }, { id: '67890' }],
}),
),
),
);
const values = await collectAsyncIterable(
client.getGroupUserMembers('group-id'),
);
expect(values).toEqual([{ id: '12345' }, { id: '67890' }]);
});
it('should load organization', async () => {
worker.use(
rest.get('https://example.com/organization/tentant-id', (_, res, ctx) =>
plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts
+21 -22
View File
@@ -234,28 +234,6 @@ export class MicrosoftGraphClient {
});
}
/**
* Get {@link https://docs.microsoft.com/en-us/graph/api/resources/user | User}
* from Graph API
*
* @public
* @param userId - The unique identifier for the `User` resource
* @param query - OData Query {@link ODataQuery}
*
*/
async getUserProfile(
userId: string,
query?: ODataQuery,
): Promise<MicrosoftGraph.User> {
const response = await this.requestApi(`users/${userId}`, query);
if (response.status !== 200) {
await this.handleError('user profile', response);
}
return await response.json();
}
/**
* Get {@link https://docs.microsoft.com/en-us/graph/api/resources/profilephoto | profilePhoto}
* of `User` from Graph API with size limit
@@ -360,6 +338,27 @@ export class MicrosoftGraphClient {
);
}
/**
* Get a collection of
* {@link https://docs.microsoft.com/en-us/graph/api/resources/user | User}
* belonging to a `Group` from Graph API and return as `AsyncIterable`
* @public
* @param groupId - The unique identifier for the `Group` resource
* @param query - OData Query {@link ODataQuery}
* @param queryMode - Mode to use while querying. Some features are only available at "advanced".
*/
async *getGroupUserMembers(
groupId: string,
query?: ODataQuery,
queryMode?: 'basic' | 'advanced',
): AsyncIterable<MicrosoftGraph.User> {
yield* this.requestCollection<MicrosoftGraph.User>(
`groups/${groupId}/members/microsoft.graph.user/`,
query,
queryMode,
);
}
/**
* Get {@link https://docs.microsoft.com/en-us/graph/api/resources/organization | Organization}
* from Graph API
plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts
+34 -44
View File
@@ -61,9 +61,9 @@ function group(data: Partial<GroupEntity>): GroupEntity {
describe('read microsoft graph', () => {
const client: jest.Mocked<MicrosoftGraphClient> = {
getUsers: jest.fn(),
getUserProfile: jest.fn(),
getGroups: jest.fn(),
getGroupMembers: jest.fn(),
getGroupUserMembers: jest.fn(),
getUserPhotoWithSizeLimit: jest.fn(),
getGroupPhotoWithSizeLimit: jest.fn(),
getOrganization: jest.fn(),
@@ -78,13 +78,6 @@ describe('read microsoft graph', () => {
mail: 'user.name@example.com',
};
}
async function getExampleUserProfile() {
return {
id: 'userid',
displayName: 'User Name',
mail: 'user.name@example.com',
};
}
async function* getExampleGroups() {
yield {
id: 'groupid',
@@ -245,9 +238,8 @@ describe('read microsoft graph', () => {
describe('readMicrosoftGraphUsersInGroups', () => {
it('should read users from Groups', async () => {
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getGroupUserMembers.mockImplementation(getExampleUsers);
client.getUserProfile.mockResolvedValue(getExampleUserProfile());
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
@@ -281,19 +273,19 @@ describe('read microsoft graph', () => {
expect(client.getGroups).toHaveBeenCalledWith(
{
filter: 'securityEnabled eq true',
select: ['id', 'displayName'],
top: 999,
},
undefined,
);
expect(client.getGroupMembers).toHaveBeenCalledTimes(1);
expect(client.getGroupMembers).toHaveBeenCalledWith('groupid', {
top: 999,
});
expect(client.getUserProfile).toHaveBeenCalledTimes(1);
expect(client.getUserProfile).toHaveBeenCalledWith('userid', {
expand: undefined,
});
expect(client.getGroupUserMembers).toHaveBeenCalledTimes(1);
expect(client.getGroupUserMembers).toHaveBeenCalledWith(
'groupid',
{ top: 999 },
undefined,
);
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledTimes(1);
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledWith(
'userid',
@@ -303,13 +295,8 @@ describe('read microsoft graph', () => {
it('should read users from Groups with advanced query mode', async () => {
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getGroupUserMembers.mockImplementation(getExampleUsers);
client.getUserProfile.mockResolvedValue({
id: 'userid',
displayName: 'User Name',
mail: 'user.name@example.com',
});
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
@@ -344,19 +331,19 @@ describe('read microsoft graph', () => {
expect(client.getGroups).toHaveBeenCalledWith(
{
filter: 'securityEnabled eq true',
select: ['id', 'displayName'],
top: 999,
},
'advanced',
);
expect(client.getGroupMembers).toHaveBeenCalledTimes(1);
expect(client.getGroupMembers).toHaveBeenCalledWith('groupid', {
top: 999,
});
expect(client.getUserProfile).toHaveBeenCalledTimes(1);
expect(client.getUserProfile).toHaveBeenCalledWith('userid', {
expand: undefined,
});
expect(client.getGroupUserMembers).toHaveBeenCalledTimes(1);
expect(client.getGroupUserMembers).toHaveBeenCalledWith(
'groupid',
{ top: 999 },
'advanced',
);
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledTimes(1);
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledWith(
'userid',
@@ -366,8 +353,8 @@ describe('read microsoft graph', () => {
it('should read users with userExpand, groupExpand and custom transformer', async () => {
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getUserProfile.mockResolvedValue(getExampleUserProfile());
client.getGroupUserMembers.mockImplementation(getExampleUsers);
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
@@ -399,19 +386,22 @@ describe('read microsoft graph', () => {
{
expand: 'member',
filter: 'securityEnabled eq true',
select: ['id', 'displayName'],
top: 999,
},
undefined,
);
expect(client.getGroupUserMembers).toHaveBeenCalledTimes(1);
expect(client.getGroupUserMembers).toHaveBeenCalledWith(
'groupid',
{
expand: 'manager',
top: 999,
},
undefined,
);
expect(client.getGroupMembers).toHaveBeenCalledTimes(1);
expect(client.getGroupMembers).toHaveBeenCalledWith('groupid', {
top: 999,
});
expect(client.getUserProfile).toHaveBeenCalledTimes(1);
expect(client.getUserProfile).toHaveBeenCalledWith('userid', {
expand: 'manager',
});
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledTimes(1);
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledWith(
'userid',
@@ -1001,13 +991,13 @@ describe('read microsoft graph', () => {
});
client.getUsers.mockImplementation(getExampleUsers);
client.getUserProfile.mockImplementation(getExampleUserProfile);
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getGroupUserMembers.mockImplementation(getExampleUsers);
client.getGroupPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
@@ -1023,6 +1013,7 @@ describe('read microsoft graph', () => {
expect(client.getGroups).toHaveBeenCalledWith(
{
filter: 'name eq backstage-group',
select: ['id', 'displayName'],
top: 999,
},
undefined,
@@ -1034,7 +1025,6 @@ describe('read microsoft graph', () => {
},
undefined,
);
expect(client.getUserProfile).toHaveBeenCalledTimes(1);
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledTimes(1);
});
});
plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts
+90 -93
View File
@@ -98,13 +98,7 @@ export async function readMicrosoftGraphUsers(
): Promise<{
users: UserEntity[]; // With all relations empty
}> {
const users: UserEntity[] = [];
const limiter = limiterFactory(10);
const transformer = options.transformer ?? defaultUserTransformer;
const promises: Promise<void>[] = [];
for await (const user of client.getUsers(
const users = client.getUsers(
{
filter: options.userFilter,
expand: options.userExpand,
@@ -112,37 +106,16 @@ export async function readMicrosoftGraphUsers(
top: PAGE_SIZE,
},
options.queryMode,
)) {
// Process all users in parallel, otherwise it can take quite some time
promises.push(
limiter(async () => {
let userPhoto;
try {
userPhoto = await client.getUserPhotoWithSizeLimit(
user.id!,
// We are limiting the photo size, as users with full resolution photos
// can make the Backstage API slow
120,
);
} catch (e) {
options.logger.warn(`Unable to load photo for ${user.id}, ${e}`);
}
);
const entity = await transformer(user, userPhoto);
if (!entity) {
return;
}
users.push(entity);
}),
);
}
// Wait for all users and photos to be downloaded
await Promise.all(promises);
return { users };
return {
users: await transformUsers(
client,
users,
options.logger,
options.transformer,
),
};
}
export async function readMicrosoftGraphUsersInGroups(
@@ -160,21 +133,17 @@ export async function readMicrosoftGraphUsersInGroups(
): Promise<{
users: UserEntity[]; // With all relations empty
}> {
const users: UserEntity[] = [];
const limiter = limiterFactory(10);
const transformer = options.transformer ?? defaultUserTransformer;
const userGroupMemberPromises: Promise<void>[] = [];
const userPromises: Promise<void>[] = [];
const groupMemberUsers: Set<string> = new Set();
const userGroupMembers = new Map<string, MicrosoftGraph.User>();
for await (const group of client.getGroups(
{
expand: options.groupExpand,
search: options.userGroupMemberSearch,
filter: options.userGroupMemberFilter,
select: ['id', 'displayName'],
top: PAGE_SIZE,
},
options.queryMode,
@@ -182,17 +151,23 @@ export async function readMicrosoftGraphUsersInGroups(
// Process all groups in parallel, otherwise it can take quite some time
userGroupMemberPromises.push(
limiter(async () => {
for await (const member of client.getGroupMembers(group.id!, {
top: PAGE_SIZE,
})) {
if (!member.id) {
continue;
}
if (member['@odata.type'] === '#microsoft.graph.user') {
groupMemberUsers.add(member.id);
}
let groupMemberCount = 0;
for await (const user of client.getGroupUserMembers(
group.id!,
{
expand: options.userExpand,
top: PAGE_SIZE,
},
options.queryMode,
)) {
userGroupMembers.set(user.id!, user);
groupMemberCount++;
}
options.logger.debug('Read users from group', {
groupId: group.id,
groupName: group.displayName,
memberCount: groupMemberCount,
});
}),
);
}
@@ -200,47 +175,19 @@ export async function readMicrosoftGraphUsersInGroups(
// Wait for all group members
await Promise.all(userGroupMemberPromises);
options.logger.info(`groupMemberUsers ${groupMemberUsers.size}`);
for (const userId of groupMemberUsers) {
// Process all users in parallel, otherwise it can take quite some time
userPromises.push(
limiter(async () => {
let user;
let userPhoto;
try {
user = await client.getUserProfile(userId, {
expand: options.userExpand,
});
} catch (e) {
options.logger.warn(`Unable to load user for ${userId}, ${e}`);
}
if (user) {
try {
userPhoto = await client.getUserPhotoWithSizeLimit(
user.id!,
// We are limiting the photo size, as users with full resolution photos
// can make the Backstage API slow
120,
);
} catch (e) {
options.logger.warn(`Unable to load userphoto for ${userId}, ${e}`);
}
options.logger.info('Read users from group membership', {
groupCount: userGroupMemberPromises.length,
userCount: userGroupMembers.size,
});
const entity = await transformer(user, userPhoto);
if (!entity) {
return;
}
users.push(entity);
}
}),
);
}
// Wait for all users and photos to be downloaded
await Promise.all(userPromises);
return { users };
return {
users: await transformUsers(
client,
userGroupMembers.values(),
options.logger,
options.transformer,
),
};
}
/**
@@ -602,6 +549,56 @@ export async function readMicrosoftGraphOrg(
return { users, groups };
}
async function transformUsers(
client: MicrosoftGraphClient,
users: Iterable<MicrosoftGraph.User> | AsyncIterable<MicrosoftGraph.User>,
logger: Logger,
transformer?: UserTransformer,
) {
const limiter = limiterFactory(10);
const resolvedTransformer = transformer ?? defaultUserTransformer;
const promises: Promise<void>[] = [];
const entities: UserEntity[] = [];
// Process all users in parallel, otherwise it can take quite some time
for await (const user of users) {
promises.push(
limiter(async () => {
let userPhoto;
try {
userPhoto = await client.getUserPhotoWithSizeLimit(
user.id!,
// We are limiting the photo size, as users with full resolution photos
// can make the Backstage API slow
120,
);
} catch (e) {
logger.warn(`Unable to load user photo for`, {
user: user.id,
error: e,
});
}
const entity = await resolvedTransformer(user, userPhoto);
if (entity) {
entities.push(entity);
}
}),
);
}
// Wait for all users and photos to be downloaded
await Promise.all(promises);
logger.debug('Finished transforming users', {
microsoftUserCount: promises.length,
backstageUserCount: entities.length,
});
return entities;
}
function ensureItem(
target: Map<string, Set<string>>,
key: string,