From a0976784752311487534ce9bbdcbc42741edf311 Mon Sep 17 00:00:00 2001 From: Guilherme Oenning Date: Tue, 22 Feb 2022 10:03:20 +0000 Subject: [PATCH 1/5] add configuration to use search criteria to select groups Signed-off-by: Guilherme Oenning --- .changeset/tender-dogs-behave.md | 5 ++++ .../api-report.md | 5 ++++ .../config.d.ts | 12 ++++++++ .../src/microsoftGraph/client.ts | 10 +++++++ .../src/microsoftGraph/config.ts | 29 +++++++++++++++---- .../src/microsoftGraph/read.ts | 8 +++++ .../MicrosoftGraphOrgEntityProvider.ts | 2 ++ .../MicrosoftGraphOrgReaderProcessor.ts | 2 ++ 8 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 .changeset/tender-dogs-behave.md diff --git a/.changeset/tender-dogs-behave.md b/.changeset/tender-dogs-behave.md new file mode 100644 index 0000000000..f59842c70a --- /dev/null +++ b/.changeset/tender-dogs-behave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add configuration to use search criteria to select groups diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index bb442aa544..9d98b12249 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -153,7 +153,9 @@ export type MicrosoftGraphProviderConfig = { userFilter?: string; userExpand?: string[]; userGroupMemberFilter?: string; + userGroupMemberSearch?: string; groupFilter?: string; + groupSearch?: string; }; // @public @@ -161,6 +163,7 @@ export function normalizeEntityName(name: string): string; // @public export type ODataQuery = { + search?: string; filter?: string; expand?: string[]; select?: string[]; @@ -183,7 +186,9 @@ export function readMicrosoftGraphOrg( options: { userExpand?: string[]; userFilter?: string; + userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupSearch?: string; groupFilter?: string; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; diff --git a/plugins/catalog-backend-module-msgraph/config.d.ts b/plugins/catalog-backend-module-msgraph/config.d.ts index c7eb5ae8e4..d47abf60bb 100644 --- a/plugins/catalog-backend-module-msgraph/config.d.ts +++ b/plugins/catalog-backend-module-msgraph/config.d.ts @@ -73,12 +73,24 @@ export interface Config { * E.g. "securityEnabled eq false and mailEnabled eq true" */ groupFilter?: string; + /** + * The search criteria to apply to extract users by groups memberships. + * + * E.g. "\"displayName:-team\"" would only match groups which contain '-team' + */ + groupSearch?: string; /** * The filter to apply to extract users by groups memberships. * * E.g. "displayName eq 'Backstage Users'" */ userGroupMemberFilter?: string; + /** + * The search criteria to apply to extract groups. + * + * E.g. "\"displayName:-team\"" would only match groups which contain '-team' + */ + userGroupMemberSearch?: string; }>; }; }; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts index 827c9fc14d..8814c9d662 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts @@ -27,6 +27,10 @@ import { MicrosoftGraphProviderConfig } from './config'; * @public */ export type ODataQuery = { + /** + * search resources within a collection matching a free-text search expression. + */ + search?: string; /** * filter a collection of resources */ @@ -135,6 +139,7 @@ export class MicrosoftGraphClient { async requestApi(path: string, query?: ODataQuery): Promise { const queryString = qs.stringify( { + $search: query?.search, $filter: query?.filter, $select: query?.select?.join(','), $expand: query?.expand?.join(','), @@ -167,6 +172,11 @@ export class MicrosoftGraphClient { return await fetch(url, { headers: { Authorization: `Bearer ${token.accessToken}`, + + // Eventual consistency is required to use $search. + // Groups/Users are not changed that frequently to require strong consistency + // If a new user/group is not found, it'll eventually be imported on a subsequent call + ConsistencyLevel: 'eventual' }, }); } diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index c401bc2553..8d72833475 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -64,12 +64,24 @@ export type MicrosoftGraphProviderConfig = { * E.g. "displayName eq 'Backstage Users'" */ userGroupMemberFilter?: string; + /** + * The search criteria to apply to extract users by groups memberships. + * + * E.g. "\"displayName:-team\"" would only match groups which contain '-team' + */ + userGroupMemberSearch?: string; /** * The filter to apply to extract groups. * * E.g. "securityEnabled eq false and mailEnabled eq true" */ groupFilter?: string; + /** + * The search criteria to apply to extract groups. + * + * E.g. "\"displayName:-team\"" would only match groups which contain '-team' + */ + groupSearch?: string; }; /** @@ -93,18 +105,21 @@ export function readMicrosoftGraphConfig( : 'https://login.microsoftonline.com'; const tenantId = providerConfig.getString('tenantId'); const clientId = providerConfig.getString('clientId'); - const clientSecret = providerConfig.getString('clientSecret'); - const userFilter = providerConfig.getOptionalString('userFilter'); - const userGroupMemberFilter = providerConfig.getOptionalString( - 'userGroupMemberFilter', - ); - const groupFilter = providerConfig.getOptionalString('groupFilter'); + const clientSecret = providerConfig.getString("clientSecret"); + const userFilter = providerConfig.getOptionalString("userFilter"); + const userGroupMemberFilter = providerConfig.getOptionalString("userGroupMemberFilter"); + const userGroupMemberSearch = providerConfig.getOptionalString("userGroupMemberSearch"); + const groupFilter = providerConfig.getOptionalString("groupFilter"); + const groupSearch = providerConfig.getOptionalString("groupSearch"); if (userFilter && userGroupMemberFilter) { throw new Error( `userFilter and userGroupMemberFilter are mutually exclusive, only one can be specified.`, ); } + if (userFilter && userGroupMemberSearch) { + throw new Error(`userGroupMemberSearch cannot be specified when userFilter is defined.`); + } providers.push({ target, @@ -114,7 +129,9 @@ export function readMicrosoftGraphConfig( clientSecret, userFilter, userGroupMemberFilter, + userGroupMemberSearch, groupFilter, + groupSearch }); } diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index 58e60ab76a..f862fd5ace 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -137,6 +137,7 @@ export async function readMicrosoftGraphUsers( export async function readMicrosoftGraphUsersInGroups( client: MicrosoftGraphClient, options: { + userGroupMemberSearch?: string; userGroupMemberFilter?: string; transformer?: UserTransformer; logger: Logger; @@ -155,6 +156,7 @@ export async function readMicrosoftGraphUsersInGroups( const groupMemberUsers: Set = new Set(); for await (const group of client.getGroups({ + search: options?.userGroupMemberSearch, filter: options?.userGroupMemberFilter, })) { // Process all groups in parallel, otherwise it can take quite some time @@ -324,6 +326,7 @@ export async function readMicrosoftGraphGroups( client: MicrosoftGraphClient, tenantId: string, options?: { + groupSearch?: string; groupFilter?: string; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; @@ -351,6 +354,7 @@ export async function readMicrosoftGraphGroups( const promises: Promise[] = []; for await (const group of client.getGroups({ + search: options?.groupSearch, filter: options?.groupFilter, })) { // Process all groups in parallel, otherwise it can take quite some time @@ -504,7 +508,9 @@ export async function readMicrosoftGraphOrg( options: { userExpand?: string[]; userFilter?: string; + userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupSearch?: string; groupFilter?: string; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; @@ -519,6 +525,7 @@ export async function readMicrosoftGraphOrg( client, { userGroupMemberFilter: options.userGroupMemberFilter, + userGroupMemberSearch: options.userGroupMemberSearch, transformer: options.userTransformer, logger: options.logger, }, @@ -535,6 +542,7 @@ export async function readMicrosoftGraphOrg( } const { groups, rootGroup, groupMember, groupMemberOf } = await readMicrosoftGraphGroups(client, tenantId, { + groupSearch: options?.groupSearch, groupFilter: options?.groupFilter, groupTransformer: options?.groupTransformer, organizationTransformer: options?.organizationTransformer, diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 92836893f0..7d27e87d73 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -123,7 +123,9 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { { userFilter: provider.userFilter, userGroupMemberFilter: provider.userGroupMemberFilter, + userGroupMemberSearch: provider.userGroupMemberSearch, groupFilter: provider.groupFilter, + groupSearch: provider.groupSearch, groupTransformer: this.options.groupTransformer, userTransformer: this.options.userTransformer, organizationTransformer: this.options.organizationTransformer, diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 78b2922f60..2a7776f449 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -108,7 +108,9 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { userExpand: provider.userExpand, userFilter: provider.userFilter, userGroupMemberFilter: provider.userGroupMemberFilter, + userGroupMemberSearch: provider.userGroupMemberSearch, groupFilter: provider.groupFilter, + groupSearch: provider.groupSearch, userTransformer: this.userTransformer, groupTransformer: this.groupTransformer, organizationTransformer: this.organizationTransformer, From 981a841e5f6673c0439989ced7d1dd2833b907d6 Mon Sep 17 00:00:00 2001 From: Guilherme Oenning Date: Tue, 22 Feb 2022 10:21:23 +0000 Subject: [PATCH 2/5] fix prettier Signed-off-by: Guilherme Oenning --- .../src/microsoftGraph/client.ts | 2 +- .../src/microsoftGraph/config.ts | 22 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts index 8814c9d662..8a9b2fd161 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts @@ -176,7 +176,7 @@ export class MicrosoftGraphClient { // Eventual consistency is required to use $search. // Groups/Users are not changed that frequently to require strong consistency // If a new user/group is not found, it'll eventually be imported on a subsequent call - ConsistencyLevel: 'eventual' + ConsistencyLevel: 'eventual', }, }); } diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index 8d72833475..6809144f80 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -105,12 +105,16 @@ export function readMicrosoftGraphConfig( : 'https://login.microsoftonline.com'; const tenantId = providerConfig.getString('tenantId'); const clientId = providerConfig.getString('clientId'); - const clientSecret = providerConfig.getString("clientSecret"); - const userFilter = providerConfig.getOptionalString("userFilter"); - const userGroupMemberFilter = providerConfig.getOptionalString("userGroupMemberFilter"); - const userGroupMemberSearch = providerConfig.getOptionalString("userGroupMemberSearch"); - const groupFilter = providerConfig.getOptionalString("groupFilter"); - const groupSearch = providerConfig.getOptionalString("groupSearch"); + const clientSecret = providerConfig.getString('clientSecret'); + const userFilter = providerConfig.getOptionalString('userFilter'); + const userGroupMemberFilter = providerConfig.getOptionalString( + 'userGroupMemberFilter', + ); + const userGroupMemberSearch = providerConfig.getOptionalString( + 'userGroupMemberSearch', + ); + const groupFilter = providerConfig.getOptionalString('groupFilter'); + const groupSearch = providerConfig.getOptionalString('groupSearch'); if (userFilter && userGroupMemberFilter) { throw new Error( @@ -118,7 +122,9 @@ export function readMicrosoftGraphConfig( ); } if (userFilter && userGroupMemberSearch) { - throw new Error(`userGroupMemberSearch cannot be specified when userFilter is defined.`); + throw new Error( + `userGroupMemberSearch cannot be specified when userFilter is defined.`, + ); } providers.push({ @@ -131,7 +137,7 @@ export function readMicrosoftGraphConfig( userGroupMemberFilter, userGroupMemberSearch, groupFilter, - groupSearch + groupSearch, }); } From 09f3666785c7d74f3c4847f1a633a13d9457f928 Mon Sep 17 00:00:00 2001 From: Guilherme Oenning Date: Tue, 22 Feb 2022 15:06:05 +0000 Subject: [PATCH 3/5] conditionally add consistency level Signed-off-by: Guilherme Oenning --- .../src/microsoftGraph/client.ts | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts index 8a9b2fd161..457a83c703 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts @@ -106,7 +106,14 @@ export class MicrosoftGraphClient { path: string, query?: ODataQuery, ): AsyncIterable { - let response = await this.requestApi(path, query); + + const headers: Record = query?.search ? { + // Eventual consistency is required to use $search. + // If a new user/group is not found, it'll eventually be imported on a subsequent read + ConsistencyLevel: 'eventual', + } : {} + + let response = await this.requestApi(path, query, headers); for (;;) { if (response.status !== 200) { @@ -125,7 +132,7 @@ export class MicrosoftGraphClient { return; } - response = await this.requestRaw(result['@odata.nextLink']); + response = await this.requestRaw(result['@odata.nextLink'], headers); } } @@ -135,8 +142,9 @@ export class MicrosoftGraphClient { * @public * @param path - Resource in Microsoft Graph * @param query - OData Query {@link ODataQuery} + * @param headers - optional HTTP headers */ - async requestApi(path: string, query?: ODataQuery): Promise { + async requestApi(path: string, query?: ODataQuery, headers?: Record): Promise { const queryString = qs.stringify( { $search: query?.search, @@ -151,15 +159,16 @@ export class MicrosoftGraphClient { }, ); - return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`); + return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`, headers); } /** * Makes a HTTP call to Graph API with token * * @param url - HTTP Endpoint of Graph API + * @param headers - optional HTTP headers */ - async requestRaw(url: string): Promise { + async requestRaw(url: string, headers?: Record): Promise { // Make sure that we always have a valid access token (might be cached) const token = await this.pca.acquireTokenByClientCredential({ scopes: ['https://graph.microsoft.com/.default'], @@ -171,12 +180,8 @@ export class MicrosoftGraphClient { return await fetch(url, { headers: { + ...headers, Authorization: `Bearer ${token.accessToken}`, - - // Eventual consistency is required to use $search. - // Groups/Users are not changed that frequently to require strong consistency - // If a new user/group is not found, it'll eventually be imported on a subsequent call - ConsistencyLevel: 'eventual', }, }); } From 21fb25f1c37fe6824afe2867c1d549f16c2a2c47 Mon Sep 17 00:00:00 2001 From: Guilherme Oenning Date: Wed, 23 Feb 2022 13:49:53 +0000 Subject: [PATCH 4/5] fix formatting Signed-off-by: Guilherme Oenning --- .../src/microsoftGraph/client.ts | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts index 457a83c703..db749a536e 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts @@ -106,12 +106,13 @@ export class MicrosoftGraphClient { path: string, query?: ODataQuery, ): AsyncIterable { - - const headers: Record = query?.search ? { - // Eventual consistency is required to use $search. - // If a new user/group is not found, it'll eventually be imported on a subsequent read - ConsistencyLevel: 'eventual', - } : {} + const headers: Record = query?.search + ? { + // Eventual consistency is required to use $search. + // If a new user/group is not found, it'll eventually be imported on a subsequent read + ConsistencyLevel: 'eventual', + } + : {}; let response = await this.requestApi(path, query, headers); @@ -144,7 +145,11 @@ export class MicrosoftGraphClient { * @param query - OData Query {@link ODataQuery} * @param headers - optional HTTP headers */ - async requestApi(path: string, query?: ODataQuery, headers?: Record): Promise { + async requestApi( + path: string, + query?: ODataQuery, + headers?: Record, + ): Promise { const queryString = qs.stringify( { $search: query?.search, @@ -159,7 +164,10 @@ export class MicrosoftGraphClient { }, ); - return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`, headers); + return await this.requestRaw( + `${this.baseUrl}/${path}${queryString}`, + headers, + ); } /** @@ -168,7 +176,10 @@ export class MicrosoftGraphClient { * @param url - HTTP Endpoint of Graph API * @param headers - optional HTTP headers */ - async requestRaw(url: string, headers?: Record): Promise { + async requestRaw( + url: string, + headers?: Record, + ): Promise { // Make sure that we always have a valid access token (might be cached) const token = await this.pca.acquireTokenByClientCredential({ scopes: ['https://graph.microsoft.com/.default'], From 918c4df3c91781b873e3d25ccd9df211c171d0f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 23 Feb 2022 16:20:08 +0100 Subject: [PATCH 5/5] api report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- plugins/catalog-backend-module-msgraph/api-report.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 9d98b12249..b83567b982 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -80,9 +80,16 @@ export class MicrosoftGraphClient { ): Promise; getUserProfile(userId: string): Promise; getUsers(query?: ODataQuery): AsyncIterable; - requestApi(path: string, query?: ODataQuery): Promise; + requestApi( + path: string, + query?: ODataQuery, + headers?: Record, + ): Promise; requestCollection(path: string, query?: ODataQuery): AsyncIterable; - requestRaw(url: string): Promise; + requestRaw( + url: string, + headers?: Record, + ): Promise; } // @public