From a0b80e7bce9120e696086f1f8c0498ef6e980c7e Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 14 Jul 2023 11:43:47 +0100 Subject: [PATCH 01/33] Add users and groups ingestion for gitlab.com Signed-off-by: angom1 --- .../src/lib/client.ts | 165 +++++++++++++++++- .../src/lib/types.ts | 52 ++++++ .../GitlabOrgDiscoveryEntityProvider.ts | 25 +-- 3 files changed, 231 insertions(+), 11 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index f0e169d8df..d2a3fa01e0 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -20,7 +20,13 @@ import { GitLabIntegrationConfig, } from '@backstage/integration'; import { Logger } from 'winston'; -import { GitLabGroup, GitLabGroupMembersResponse, GitLabUser } from './types'; +import { + GitLabGroup, + GitLabSaasUsersResponse, + GitLabSaasGroupsResponse, + GitLabGroupMembersResponse, + GitLabUser, +} from './types'; export type CommonListOptions = { [key: string]: string | number | boolean | undefined; @@ -91,6 +97,163 @@ export class GitLabClient { return this.pagedRequest(`/groups`, options); } + async listSaasUsers(groupPath: string): Promise> { + const items: GitLabUser[] = []; + let hasNextPage: boolean = false; + let endCursor: string | null = null; + + if (!groupPath) { + throw new Error( + `Missing required 'group' value in gitlab provider config.`, + ); + } + + do { + const response: GitLabSaasUsersResponse = await fetch( + `${this.config.baseUrl}/api/graphql`, + { + method: 'POST', + headers: { + ...getGitLabRequestOptions(this.config).headers, + ['Content-Type']: 'application/json', + }, + body: JSON.stringify({ + variables: { group: groupPath, endCursor }, + query: `query($group: ID!, $endCursor: String) { + group(fullPath: $group) { + groupMembers(first: 100, relations: [DESCENDANTS], after: $endCursor) { + nodes { + user { + id + username + commitEmail + name + state + webUrl + avatarUrl + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + }`, + }), + }, + ).then(r => r.json()); + if (response.errors) { + throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); + } + + if (!response.data.group?.groupMembers?.nodes) { + this.logger.warn( + `Couldn't get members under ${groupPath}. The provided token might not have sufficient permissions`, + ); + continue; + } + + const usersData = response.data.group.groupMembers.nodes; + + for (let i = 0; i < usersData.length; i++) { + const userItem = usersData[i]; + + const formattedUserResponse = { + id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), + username: userItem.user.username, + email: userItem.user.commitEmail, + name: userItem.user.name, + state: userItem.user.state, + web_url: userItem.user.webUrl, + avatar_url: userItem.user.avatarUrl, + }; + + items.push(formattedUserResponse); + } + ({ hasNextPage, endCursor } = response.data.group.groupMembers.pageInfo); + } while (hasNextPage); + return { items }; + } + + async listSaasGroups(groupPath: string): Promise> { + const items: GitLabGroup[] = []; + let hasNextPage: boolean = false; + let endCursor: string | null = null; + + if (!groupPath) { + throw new Error( + `Missing required 'group' value in gitlab provider config.`, + ); + } + + do { + const response: GitLabSaasGroupsResponse = await fetch( + `${this.config.baseUrl}/api/graphql`, + { + method: 'POST', + headers: { + ...getGitLabRequestOptions(this.config).headers, + ['Content-Type']: 'application/json', + }, + body: JSON.stringify({ + variables: { group: groupPath, endCursor }, + query: `query($group: ID!, $endCursor: String) { + group(fullPath: $group) { + descendantGroups(first: 100, after: $endCursor){ + nodes{ + id + name + description + fullPath + parent{ + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + }`, + }), + }, + ).then(r => r.json()); + if (response.errors) { + throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); + } + + if (!response.data.group?.descendantGroups.nodes) { + this.logger.warn( + `Could not get groups under ${groupPath}. The provided token might not have sufficient permissions`, + ); + continue; + } + + const groupsData = response.data.group.descendantGroups.nodes; + + for (let i = 0; i < groupsData.length; i++) { + const groupItem = groupsData[i]; + + const formattedGroupResponse = { + id: Number(groupItem.id.replace(/^gid:\/\/gitlab\/Group\//, '')), + name: groupItem.name, + description: groupItem.description, + full_path: groupItem.fullPath, + parent_id: Number( + groupItem.parent.id.replace(/^gid:\/\/gitlab\/Group\//, ''), + ), + }; + + items.push(formattedGroupResponse); + } + ({ hasNextPage, endCursor } = + response.data.group.descendantGroups.pageInfo); + } while (hasNextPage); + return { items }; + } + async getGroupMembers(groupPath: string): Promise { const memberIds = []; let hasNextPage: boolean = false; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 341e2a9d9a..a00c66b4a7 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -70,6 +70,58 @@ export type GitLabGroupMembersResponse = { }; }; +export type GitLabSaasUsersResponse = { + errors: { message: string }[]; + data: { + group: { + groupMembers: { + nodes: [ + { + user: { + id: string; + username: string; + commitEmail: string; + name: string; + state: string; + webUrl: string; + avatarUrl: string; + }; + }, + ]; + pageInfo: { + endCursor: string; + hasNextPage: boolean; + }; + }; + }; + }; +}; + +export type GitLabSaasGroupsResponse = { + errors: { message: string }[]; + data: { + group: { + descendantGroups: { + nodes: [ + { + id: string; + name: string; + description: string; + fullPath: string; + parent: { + id: string; + }; + }, + ]; + pageInfo: { + endCursor: string; + hasNextPage: false; + }; + }; + }; + }; +}; + export type GitlabProviderConfig = { host: string; group: string; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 02c036f27a..5fd5094943 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -168,19 +168,24 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { logger: logger, }); - const users = paginated(options => client.listUsers(options), { - page: 1, - per_page: 100, - active: true, - }); + let groups; + let users; - const groups = paginated( - options => client.listGroups(options), - { + if (client.isSelfManaged()) { + groups = paginated(options => client.listGroups(options), { page: 1, per_page: 100, - }, - ); + }); + + users = paginated(options => client.listUsers(options), { + page: 1, + per_page: 100, + active: true, + }); + } else { + groups = (await client.listSaasGroups(this.config.group)).items; + users = (await client.listSaasUsers(this.config.group)).items; + } const idMappedUser: { [userId: number]: GitLabUser } = {}; From d2f13494d84013a1b85e55a7cc494d4c0a03f0f5 Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 14 Jul 2023 16:11:43 +0100 Subject: [PATCH 02/33] Update error handling and documentation Signed-off-by: angom1 --- docs/integrations/gitlab/org.md | 4 +++- .../src/lib/client.ts | 14 +------------- .../providers/GitlabOrgDiscoveryEntityProvider.ts | 6 ++++++ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index d022641ab6..6c154938ee 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -58,6 +58,8 @@ amount of data, this can take significant time and resources. The token used must have the `read_api` scope, and the Users and Groups fetched will be those visible to the account which provisioned the token. +**NOTE**: If any groups that are being ingested are empty groups and the account which provisioned the token is shared at a higher level group via group sharing and you don't see the expected number of Groups in the catalog you may be hitting this [GitLab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). + ```yaml catalog: providers: @@ -65,7 +67,7 @@ catalog: yourProviderId: host: gitlab.com orgEnabled: true - group: org/teams # Optional. Must not end with slash. Accepts only groups under the provided path (which will be stripped) + group: org/teams # Required for gitlab.com. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything ``` diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index d2a3fa01e0..1e5e3ea32b 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -102,12 +102,6 @@ export class GitLabClient { let hasNextPage: boolean = false; let endCursor: string | null = null; - if (!groupPath) { - throw new Error( - `Missing required 'group' value in gitlab provider config.`, - ); - } - do { const response: GitLabSaasUsersResponse = await fetch( `${this.config.baseUrl}/api/graphql`, @@ -181,12 +175,6 @@ export class GitLabClient { let hasNextPage: boolean = false; let endCursor: string | null = null; - if (!groupPath) { - throw new Error( - `Missing required 'group' value in gitlab provider config.`, - ); - } - do { const response: GitLabSaasGroupsResponse = await fetch( `${this.config.baseUrl}/api/graphql`, @@ -226,7 +214,7 @@ export class GitLabClient { if (!response.data.group?.descendantGroups.nodes) { this.logger.warn( - `Could not get groups under ${groupPath}. The provided token might not have sufficient permissions`, + `Couldn't get groups under ${groupPath}. The provided token might not have sufficient permissions`, ); continue; } diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 5fd5094943..94d3e64a02 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -89,6 +89,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ); } + if (!providerConfig.group && providerConfig.host === 'gitlab.com') { + throw new Error( + `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`, + ); + } + if (!options.schedule && !providerConfig.schedule) { throw new Error( `No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`, From 1d2d469d31c76661defc3659c6b2db79b1fffa09 Mon Sep 17 00:00:00 2001 From: angom1 Date: Mon, 17 Jul 2023 12:04:12 +0100 Subject: [PATCH 03/33] Add test when group parameter is not set Signed-off-by: angom1 --- .../GitlabOrgDiscoveryEntityProvider.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index e29cdccc04..fea973333a 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -543,6 +543,42 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); }); + it('fail with scheduler but no group config when host is gitlab.com', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + orgEnabled: true, + }, + }, + }, + }, + }); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`, + ); + }); + it('single simple provider config with schedule in config', async () => { const schedule = new PersistingTaskRunner(); const scheduler = { From a446d9eadcc3e3d52a0a7b805228214fad32e9d2 Mon Sep 17 00:00:00 2001 From: angom1 Date: Mon, 17 Jul 2023 12:27:46 +0100 Subject: [PATCH 04/33] Fix host and url Signed-off-by: angom1 --- .../src/providers/GitlabOrgDiscoveryEntityProvider.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index fea973333a..b1d95b63fa 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -551,8 +551,8 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { integrations: { gitlab: [ { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', token: '1234', }, ], From 8f8391b768678a10a6acf300a448ed0b1bb2b771 Mon Sep 17 00:00:00 2001 From: angom1 Date: Wed, 19 Jul 2023 16:43:59 +0100 Subject: [PATCH 05/33] Add test for listSaasUsers function Signed-off-by: angom1 --- .../src/lib/client.test.ts | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index f4504e8937..61376e33b5 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -398,6 +398,149 @@ describe('GitLabClient', () => { ]); }); + describe('listSaasUsers', () => { + it('gets all users under group', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + commitEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasMembers = (await client.listSaasUsers('group1')).items; + + expect(saasMembers.length).toEqual(1); + }); + + it('gets all users with token without full permissions', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.data({ + group: {}, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasMembers = (await client.listSaasUsers('group1')).items; + + expect(saasMembers).toEqual([]); + }); + + it('rejects when GraphQL returns errors', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + await expect(() => client.listSaasUsers('group1')).rejects.toThrow( + 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', + ); + }); + it('traverses multi-page results', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((req, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: req.variables.endCursor + ? [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + commitEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ] + : [ + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + commitEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasMembers = (await client.listSaasUsers('group1')).items; + + expect(saasMembers.length).toEqual(2); + }); + }); + describe('getGroupMembers', () => { it('gets member IDs', async () => { server.use( From 80a049c361066746458ff5fa95f61f864cd2021e Mon Sep 17 00:00:00 2001 From: angom1 Date: Thu, 20 Jul 2023 10:38:45 +0100 Subject: [PATCH 06/33] fix key names Signed-off-by: angom1 --- .../src/lib/client.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 61376e33b5..c718398582 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -413,11 +413,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - commitEmail: 'user1@example.com', + email: 'user1@example.com', name: 'user1', state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', + web_url: 'user1.com', + avatar_url: 'user1', }, }, ], @@ -499,11 +499,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - commitEmail: 'user1@example.com', + email: 'user1@example.com', name: 'user1', state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', + web_url: 'user1.com', + avatar_url: 'user1', }, }, ] @@ -512,11 +512,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/2', username: 'user2', - commitEmail: 'user2@example.com', + email: 'user2@example.com', name: 'user2', state: 'active', - webUrl: 'user2.com', - avatarUrl: 'user2', + web_url: 'user2.com', + avatar_url: 'user2', }, }, ], From 3be6da71a1976820400fc299965caaf1295e2e15 Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 21 Jul 2023 12:47:17 +0100 Subject: [PATCH 07/33] Add tests for listSaasGroups Signed-off-by: angom1 --- .../src/lib/client.test.ts | 155 +++++++++++++++++- .../src/lib/client.ts | 2 +- 2 files changed, 147 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index c718398582..b702889835 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -413,11 +413,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - email: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', + webUrl: 'user1.com', + avatarUrl: 'user1', }, }, ], @@ -499,11 +499,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - email: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', + webUrl: 'user1.com', + avatarUrl: 'user1', }, }, ] @@ -512,11 +512,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/2', username: 'user2', - email: 'user2@example.com', + commitEmail: 'user2@example.com', name: 'user2', state: 'active', - web_url: 'user2.com', - avatar_url: 'user2', + webUrl: 'user2.com', + avatarUrl: 'user2', }, }, ], @@ -541,6 +541,143 @@ describe('GitLabClient', () => { }); }); + describe('listSaasGroups', () => { + it('gets all groups under root', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/1', + name: 'group1', + description: 'description1', + fullPath: 'path/user1', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasGroups = (await client.listSaasGroups('group1')).items; + + expect(saasGroups.length).toEqual(1); + }); + + it('gets all descendant groups with token without full permissions', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.data({ + group: {}, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasGroups = (await client.listSaasGroups('group1')).items; + + expect(saasGroups).toEqual([]); + }); + + it('rejects when GraphQL returns errors', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + await expect(() => client.listSaasGroups('group1')).rejects.toThrow( + 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', + ); + }); + it('traverses multi-page results', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((req, res, ctx) => + res( + ctx.data({ + group: { + descendantGroups: { + nodes: req.variables.endCursor + ? [ + { + id: 'gid://gitlab/Group/1', + name: 'group1', + description: 'description1', + fullPath: 'root/group1', + parent: { + id: '123', + }, + }, + ] + : [ + { + id: 'gid://gitlab/Group/2', + name: 'group2', + description: 'description2', + fullPath: 'root/group2', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasGroups = (await client.listSaasGroups('root')).items; + + expect(saasGroups.length).toEqual(2); + }); + }); + describe('getGroupMembers', () => { it('gets member IDs', async () => { server.use( diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 1e5e3ea32b..a015cd300a 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -212,7 +212,7 @@ export class GitLabClient { throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); } - if (!response.data.group?.descendantGroups.nodes) { + if (!response.data.group?.descendantGroups?.nodes) { this.logger.warn( `Couldn't get groups under ${groupPath}. The provided token might not have sufficient permissions`, ); From 79dbbc0307756ab5441d13e99337e7fdac37f57f Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 21 Jul 2023 13:00:58 +0100 Subject: [PATCH 08/33] Fix name in test Signed-off-by: angom1 --- plugins/catalog-backend-module-gitlab/src/lib/client.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index b702889835..14bd51d938 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -556,7 +556,7 @@ describe('GitLabClient', () => { id: 'gid://gitlab/Group/1', name: 'group1', description: 'description1', - fullPath: 'path/user1', + fullPath: 'path/group1', parent: { id: '123', }, From 096e2c909cb6bce4afcaf0f3d04634600c064d1f Mon Sep 17 00:00:00 2001 From: angom1 Date: Wed, 26 Jul 2023 15:27:32 +0100 Subject: [PATCH 09/33] Add draft test Signed-off-by: angom1 --- .../GitlabOrgDiscoveryEntityProvider.test.ts | 312 ++++++++++++++++++ 1 file changed, 312 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index b1d95b63fa..eb070a54ac 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -475,6 +475,318 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { }); }); + // it('apply full update on scheduled execution for gitlab.com', async () => { + // const config = new ConfigReader({ + // integrations: { + // gitlab: [ + // { + // host: 'gitlab.com', + // apiBaseUrl: 'https://gitlab.com/api/v4', + // token: '1234', + // }, + // ], + // }, + // catalog: { + // providers: { + // gitlab: { + // 'test-id': { + // host: 'gitlab.com', + // group: 'test-group', + // orgEnabled: true, + // }, + // }, + // }, + // }, + // }); + // const schedule = new PersistingTaskRunner(); + // const entityProviderConnection: EntityProviderConnection = { + // applyMutation: jest.fn(), + // refresh: jest.fn(), + // }; + // const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + // logger, + // schedule, + // })[0]; + // expect(provider.getProviderName()).toEqual( + // 'GitlabOrgDiscoveryEntityProvider:test-id', + // ); + + // server.use( + // graphql + // .link('https://gitlab.com/api/graphql') + // .operation( + // (_req, res, ctx) => { + // const response = + // { + // id: 123, + // default_branch: 'master', + // archived: false, + // last_activity_at: new Date().toString(), + // web_url: 'https://gitlab.com/test-group/test-repo', + // path_with_namespace: 'test-group/test-repo', + // }; + // return res(ctx.data(response)); + // }, + // ), + // graphql + // .link('https://gitlab.com/api/graphql') + // .operation((_req, res, ctx) => { + // const response = [ + // { + // id: 1, + // username: 'test1', + // name: 'Test Testit', + // state: 'active', + // avatar_url: 'https://secure.gravatar.com/', + // web_url: 'https://gitlab.com/test1', + // created_at: '2023-01-19T07:27:03.333Z', + // bio: '', + // location: null, + // public_email: null, + // skype: '', + // linkedin: '', + // twitter: '', + // website_url: '', + // organization: null, + // job_title: '', + // pronouns: null, + // bot: false, + // work_information: null, + // followers: 0, + // following: 0, + // is_followed: false, + // local_time: null, + // last_sign_in_at: '2023-01-19T07:27:49.601Z', + // confirmed_at: '2023-01-19T07:27:02.905Z', + // last_activity_on: '2023-01-19', + // email: 'test@example.com', + // theme_id: 1, + // color_scheme_id: 1, + // projects_limit: 100000, + // current_sign_in_at: '2023-01-19T09:09:10.676Z', + // identities: [], + // can_create_group: true, + // can_create_project: true, + // two_factor_enabled: false, + // external: false, + // private_profile: false, + // commit_email: 'test@example.com', + // is_admin: false, + // note: '', + // }, + // { + // id: 2, + // username: 'test2', + // name: '', + // state: 'active', + // avatar_url: '', + // web_url: 'https://gitlab.com/test2', + // created_at: '2023-01-19T07:27:03.333Z', + // bio: '', + // location: null, + // public_email: null, + // skype: '', + // linkedin: '', + // twitter: '', + // website_url: '', + // organization: null, + // job_title: '', + // pronouns: null, + // bot: false, + // work_information: null, + // followers: 0, + // following: 0, + // is_followed: false, + // local_time: null, + // last_sign_in_at: '2023-01-19T07:27:49.601Z', + // confirmed_at: '2023-01-19T07:27:02.905Z', + // last_activity_on: '2023-01-19', + // email: 'test@example.com', + // theme_id: 1, + // color_scheme_id: 1, + // projects_limit: 100000, + // current_sign_in_at: '2023-01-19T09:09:10.676Z', + // identities: [], + // can_create_group: true, + // can_create_project: true, + // two_factor_enabled: false, + // external: false, + // private_profile: false, + // commit_email: 'test@example.com', + // is_admin: false, + // note: '', + // }, + // ]; + // return res(ctx.data(response)); + // }), + // graphql + // .link('https://gitlab.com/api/graphql') + // .operation((_req, res, ctx) => { + // const response = [ + // { + // id: 1, + // web_url: 'https://gitlab.com/groups/group1', + // name: 'group1', + // path: 'group1', + // description: '', + // visibility: 'internal', + // share_with_group_lock: false, + // require_two_factor_authentication: false, + // two_factor_grace_period: 48, + // project_creation_level: 'developer', + // auto_devops_enabled: null, + // subgroup_creation_level: 'owner', + // emails_disabled: null, + // mentions_disabled: null, + // lfs_enabled: true, + // default_branch_protection: 2, + // avatar_url: null, + // request_access_enabled: false, + // full_name: '8020', + // full_path: '8020', + // created_at: '2017-06-19T06:42:34.160Z', + // parent_id: null, + // }, + // { + // id: 2, + // web_url: 'https://gitlab.com/groups/group1/group2', + // name: 'group2', + // path: 'group1/group2', + // description: 'Group2', + // visibility: 'internal', + // share_with_group_lock: false, + // require_two_factor_authentication: false, + // two_factor_grace_period: 48, + // project_creation_level: 'developer', + // auto_devops_enabled: null, + // subgroup_creation_level: 'owner', + // emails_disabled: null, + // mentions_disabled: null, + // lfs_enabled: true, + // request_access_enabled: false, + // full_name: 'group2', + // full_path: 'group1/group2', + // created_at: '2017-12-07T13:20:40.675Z', + // parent_id: null, + // }, + // ]; + // return res(ctx.data(response)); + // }), + // graphql + // .link('https://gitlab.com/api/graphql') + // .operation(async (req, res, ctx) => + // res( + // ctx.data({ + // group: { + // groupMembers: { + // nodes: + // req.variables.group === 'group1/group2' + // ? [{ user: { id: 'gid://gitlab/User/1' } }] + // : [], + // pageInfo: { + // endCursor: 'end', + // hasNextPage: false, + // }, + // }, + // }, + // }), + // ), + // ), + // ); + + // await provider.connect(entityProviderConnection); + + // const taskDef = schedule.getTasks()[0]; + // expect(taskDef.id).toEqual( + // 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', + // ); + // await (taskDef.fn as () => Promise)(); + + // const expectedEntities = [ + // { + // entity: { + // apiVersion: 'backstage.io/v1alpha1', + // kind: 'User', + // metadata: { + // annotations: { + // 'backstage.io/managed-by-location': + // 'url:https://gitlab.com/test1', + // 'backstage.io/managed-by-origin-location': + // 'url:https://gitlab.com/test1', + // 'gitlab.com/user-login': 'https://gitlab.com/test1', + // }, + // name: 'test1', + // }, + // spec: { + // memberOf: ['group1-group2'], + // profile: { + // displayName: 'Test Testit', + // email: 'test@example.com', + // picture: 'https://secure.gravatar.com/', + // }, + // }, + // }, + // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + // }, + // { + // entity: { + // apiVersion: 'backstage.io/v1alpha1', + // kind: 'User', + // metadata: { + // annotations: { + // 'backstage.io/managed-by-location': + // 'url:https://gitlab.com/test2', + // 'backstage.io/managed-by-origin-location': + // 'url:https://gitlab.com/test2', + // 'gitlab.com/user-login': 'https://gitlab.com/test2', + // }, + // name: 'test2', + // }, + // spec: { + // memberOf: [], + // profile: { + // displayName: undefined, + // email: 'test@example.com', + // picture: undefined, + // }, + // }, + // }, + // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + // }, + // { + // entity: { + // apiVersion: 'backstage.io/v1alpha1', + // kind: 'Group', + // metadata: { + // annotations: { + // 'backstage.io/managed-by-location': + // 'url:https://gitlab.com/group1/group2', + // 'backstage.io/managed-by-origin-location': + // 'url:https://gitlab.com/group1/group2', + // 'gitlab.com/team-path': 'group1/group2', + // }, + // description: 'Group2', + // name: 'group1-group2', + // }, + // spec: { + // children: [], + // profile: { + // displayName: 'group2', + // }, + // type: 'team', + // }, + // }, + // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + // }, + // ]; + + // expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + // expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + // type: 'full', + // entities: expectedEntities, + // }); + // }); + it('fail without schedule and scheduler', () => { const config = new ConfigReader({ integrations: { From f0b82fcf2603bd8689f0e3fb30b8aa5641160ce0 Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 28 Jul 2023 12:32:23 +0100 Subject: [PATCH 10/33] Update draft test Signed-off-by: angom1 --- .../src/lib/client.ts | 6 +- .../GitlabOrgDiscoveryEntityProvider.test.ts | 492 +++++++----------- 2 files changed, 189 insertions(+), 309 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index a015cd300a..e37d7f27dc 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -113,7 +113,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query($group: ID!, $endCursor: String) { + query: `query listSaasUsers($group: ID!, $endCursor: String) { group(fullPath: $group) { groupMembers(first: 100, relations: [DESCENDANTS], after: $endCursor) { nodes { @@ -186,7 +186,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query($group: ID!, $endCursor: String) { + query: `query listSaasGroups($group: ID!, $endCursor: String) { group(fullPath: $group) { descendantGroups(first: 100, after: $endCursor){ nodes{ @@ -257,7 +257,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query($group: ID!, $endCursor: String) { + query: `query getGroupMembers($group: ID!, $endCursor: String) { group(fullPath: $group) { groupMembers(first: 100, relations: [DIRECT], after: $endCursor) { nodes { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index eb070a54ac..b6c62be5a2 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -475,317 +475,197 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { }); }); - // it('apply full update on scheduled execution for gitlab.com', async () => { - // const config = new ConfigReader({ - // integrations: { - // gitlab: [ - // { - // host: 'gitlab.com', - // apiBaseUrl: 'https://gitlab.com/api/v4', - // token: '1234', - // }, - // ], - // }, - // catalog: { - // providers: { - // gitlab: { - // 'test-id': { - // host: 'gitlab.com', - // group: 'test-group', - // orgEnabled: true, - // }, - // }, - // }, - // }, - // }); - // const schedule = new PersistingTaskRunner(); - // const entityProviderConnection: EntityProviderConnection = { - // applyMutation: jest.fn(), - // refresh: jest.fn(), - // }; - // const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - // logger, - // schedule, - // })[0]; - // expect(provider.getProviderName()).toEqual( - // 'GitlabOrgDiscoveryEntityProvider:test-id', - // ); + it('apply full update on scheduled execution for gitlab.com', async () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + group: 'group1', + orgEnabled: true, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); - // server.use( - // graphql - // .link('https://gitlab.com/api/graphql') - // .operation( - // (_req, res, ctx) => { - // const response = - // { - // id: 123, - // default_branch: 'master', - // archived: false, - // last_activity_at: new Date().toString(), - // web_url: 'https://gitlab.com/test-group/test-repo', - // path_with_namespace: 'test-group/test-repo', - // }; - // return res(ctx.data(response)); - // }, - // ), - // graphql - // .link('https://gitlab.com/api/graphql') - // .operation((_req, res, ctx) => { - // const response = [ - // { - // id: 1, - // username: 'test1', - // name: 'Test Testit', - // state: 'active', - // avatar_url: 'https://secure.gravatar.com/', - // web_url: 'https://gitlab.com/test1', - // created_at: '2023-01-19T07:27:03.333Z', - // bio: '', - // location: null, - // public_email: null, - // skype: '', - // linkedin: '', - // twitter: '', - // website_url: '', - // organization: null, - // job_title: '', - // pronouns: null, - // bot: false, - // work_information: null, - // followers: 0, - // following: 0, - // is_followed: false, - // local_time: null, - // last_sign_in_at: '2023-01-19T07:27:49.601Z', - // confirmed_at: '2023-01-19T07:27:02.905Z', - // last_activity_on: '2023-01-19', - // email: 'test@example.com', - // theme_id: 1, - // color_scheme_id: 1, - // projects_limit: 100000, - // current_sign_in_at: '2023-01-19T09:09:10.676Z', - // identities: [], - // can_create_group: true, - // can_create_project: true, - // two_factor_enabled: false, - // external: false, - // private_profile: false, - // commit_email: 'test@example.com', - // is_admin: false, - // note: '', - // }, - // { - // id: 2, - // username: 'test2', - // name: '', - // state: 'active', - // avatar_url: '', - // web_url: 'https://gitlab.com/test2', - // created_at: '2023-01-19T07:27:03.333Z', - // bio: '', - // location: null, - // public_email: null, - // skype: '', - // linkedin: '', - // twitter: '', - // website_url: '', - // organization: null, - // job_title: '', - // pronouns: null, - // bot: false, - // work_information: null, - // followers: 0, - // following: 0, - // is_followed: false, - // local_time: null, - // last_sign_in_at: '2023-01-19T07:27:49.601Z', - // confirmed_at: '2023-01-19T07:27:02.905Z', - // last_activity_on: '2023-01-19', - // email: 'test@example.com', - // theme_id: 1, - // color_scheme_id: 1, - // projects_limit: 100000, - // current_sign_in_at: '2023-01-19T09:09:10.676Z', - // identities: [], - // can_create_group: true, - // can_create_project: true, - // two_factor_enabled: false, - // external: false, - // private_profile: false, - // commit_email: 'test@example.com', - // is_admin: false, - // note: '', - // }, - // ]; - // return res(ctx.data(response)); - // }), - // graphql - // .link('https://gitlab.com/api/graphql') - // .operation((_req, res, ctx) => { - // const response = [ - // { - // id: 1, - // web_url: 'https://gitlab.com/groups/group1', - // name: 'group1', - // path: 'group1', - // description: '', - // visibility: 'internal', - // share_with_group_lock: false, - // require_two_factor_authentication: false, - // two_factor_grace_period: 48, - // project_creation_level: 'developer', - // auto_devops_enabled: null, - // subgroup_creation_level: 'owner', - // emails_disabled: null, - // mentions_disabled: null, - // lfs_enabled: true, - // default_branch_protection: 2, - // avatar_url: null, - // request_access_enabled: false, - // full_name: '8020', - // full_path: '8020', - // created_at: '2017-06-19T06:42:34.160Z', - // parent_id: null, - // }, - // { - // id: 2, - // web_url: 'https://gitlab.com/groups/group1/group2', - // name: 'group2', - // path: 'group1/group2', - // description: 'Group2', - // visibility: 'internal', - // share_with_group_lock: false, - // require_two_factor_authentication: false, - // two_factor_grace_period: 48, - // project_creation_level: 'developer', - // auto_devops_enabled: null, - // subgroup_creation_level: 'owner', - // emails_disabled: null, - // mentions_disabled: null, - // lfs_enabled: true, - // request_access_enabled: false, - // full_name: 'group2', - // full_path: 'group1/group2', - // created_at: '2017-12-07T13:20:40.675Z', - // parent_id: null, - // }, - // ]; - // return res(ctx.data(response)); - // }), - // graphql - // .link('https://gitlab.com/api/graphql') - // .operation(async (req, res, ctx) => - // res( - // ctx.data({ - // group: { - // groupMembers: { - // nodes: - // req.variables.group === 'group1/group2' - // ? [{ user: { id: 'gid://gitlab/User/1' } }] - // : [], - // pageInfo: { - // endCursor: 'end', - // hasNextPage: false, - // }, - // }, - // }, - // }), - // ), - // ), - // ); + server.use( + graphql + .link('https://gitlab.com/api/graphql') + .query('listSaasGroups', async (_, res, ctx) => + res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/456', + name: 'group2', + description: 'Group2', + fullPath: 'group1/group2', + parent: { + id: 'gid://gitlab/Group/123', + }, + }, + { + id: 'gid://gitlab/Group/789', + name: 'group3', + description: 'Group3', + fullPath: 'group1/group3', + parent: { + id: 'gid://gitlab/Group/123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + graphql + .link('https://gitlab.com/api/graphql') + .query('listSaasUsers', async (_, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1234', + username: 'testuser', + commitEmail: 'testuser@example.com', + state: 'active', + name: 'Test User', + webUrl: 'https://gitlab.com/testuser', + avatarUrl: 'https://secure.gravatar.com/', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + graphql + .link('https://gitlab.com/api/graphql') + .query('getGroupMembers', async (req, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: + req.variables.group === 'group1/group2' + ? [{ user: { id: 'gid://gitlab/User/1234' } }] + : [], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + ); - // await provider.connect(entityProviderConnection); + await provider.connect(entityProviderConnection); - // const taskDef = schedule.getTasks()[0]; - // expect(taskDef.id).toEqual( - // 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', - // ); - // await (taskDef.fn as () => Promise)(); + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', + ); + await (taskDef.fn as () => Promise)(); - // const expectedEntities = [ - // { - // entity: { - // apiVersion: 'backstage.io/v1alpha1', - // kind: 'User', - // metadata: { - // annotations: { - // 'backstage.io/managed-by-location': - // 'url:https://gitlab.com/test1', - // 'backstage.io/managed-by-origin-location': - // 'url:https://gitlab.com/test1', - // 'gitlab.com/user-login': 'https://gitlab.com/test1', - // }, - // name: 'test1', - // }, - // spec: { - // memberOf: ['group1-group2'], - // profile: { - // displayName: 'Test Testit', - // email: 'test@example.com', - // picture: 'https://secure.gravatar.com/', - // }, - // }, - // }, - // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - // }, - // { - // entity: { - // apiVersion: 'backstage.io/v1alpha1', - // kind: 'User', - // metadata: { - // annotations: { - // 'backstage.io/managed-by-location': - // 'url:https://gitlab.com/test2', - // 'backstage.io/managed-by-origin-location': - // 'url:https://gitlab.com/test2', - // 'gitlab.com/user-login': 'https://gitlab.com/test2', - // }, - // name: 'test2', - // }, - // spec: { - // memberOf: [], - // profile: { - // displayName: undefined, - // email: 'test@example.com', - // picture: undefined, - // }, - // }, - // }, - // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - // }, - // { - // entity: { - // apiVersion: 'backstage.io/v1alpha1', - // kind: 'Group', - // metadata: { - // annotations: { - // 'backstage.io/managed-by-location': - // 'url:https://gitlab.com/group1/group2', - // 'backstage.io/managed-by-origin-location': - // 'url:https://gitlab.com/group1/group2', - // 'gitlab.com/team-path': 'group1/group2', - // }, - // description: 'Group2', - // name: 'group1-group2', - // }, - // spec: { - // children: [], - // profile: { - // displayName: 'group2', - // }, - // type: 'team', - // }, - // }, - // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - // }, - // ]; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser', + 'gitlab.com/user-login': 'https://gitlab.com/testuser', + }, + name: 'testuser', + }, + spec: { + memberOf: ['group2'], + profile: { + displayName: 'Test User', + email: 'testuser@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/group1/group2', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/group1/group2', + 'gitlab.com/team-path': 'group1/group2', + }, + description: 'Group2', + name: 'group2', + }, + spec: { + children: [], + profile: { + displayName: 'group2', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + ]; - // expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); - // expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - // type: 'full', - // entities: expectedEntities, - // }); - // }); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); it('fail without schedule and scheduler', () => { const config = new ConfigReader({ From f85f6730a61e36d921d453fe6b9da3a414145cfd Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 28 Jul 2023 14:34:58 +0100 Subject: [PATCH 11/33] Update client tests Signed-off-by: angom1 --- .../src/lib/client.test.ts | 91 ++++++++++++++++--- 1 file changed, 77 insertions(+), 14 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 14bd51d938..b9fdf905ee 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -403,7 +403,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasUsers', async (_, res, ctx) => res( ctx.data({ group: { @@ -437,15 +437,27 @@ describe('GitLabClient', () => { }); const saasMembers = (await client.listSaasUsers('group1')).items; + const expectedSaasMember = [ + { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }, + ]; expect(saasMembers.length).toEqual(1); + expect(saasMembers).toEqual(expectedSaasMember); }); it('gets all users with token without full permissions', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasUsers', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -467,7 +479,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasUsers', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -488,7 +500,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((req, res, ctx) => + .query('listSaasUsers', async (req, res, ctx) => res( ctx.data({ group: { @@ -537,7 +549,29 @@ describe('GitLabClient', () => { const saasMembers = (await client.listSaasUsers('group1')).items; + const expectedSaasMember1 = { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }; + + const expectedSaasMember2 = { + id: 2, + username: 'user2', + email: 'user2@example.com', + name: 'user2', + state: 'active', + web_url: 'user2.com', + avatar_url: 'user2', + }; + expect(saasMembers.length).toEqual(2); + expect(saasMembers[0]).toEqual(expectedSaasMember2); + expect(saasMembers[1]).toEqual(expectedSaasMember1); }); }); @@ -546,7 +580,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasGroups', async (_, res, ctx) => res( ctx.data({ group: { @@ -579,14 +613,25 @@ describe('GitLabClient', () => { const saasGroups = (await client.listSaasGroups('group1')).items; + const expectedSaasGroup = [ + { + id: 1, + name: 'group1', + description: 'description1', + full_path: 'path/group1', + parent_id: 123, + }, + ]; + expect(saasGroups.length).toEqual(1); + expect(saasGroups).toEqual(expectedSaasGroup); }); it('gets all descendant groups with token without full permissions', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasGroups', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -608,7 +653,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasGroups', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -629,7 +674,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((req, res, ctx) => + .query('listSaasGroups', async (req, res, ctx) => res( ctx.data({ group: { @@ -640,7 +685,7 @@ describe('GitLabClient', () => { id: 'gid://gitlab/Group/1', name: 'group1', description: 'description1', - fullPath: 'root/group1', + fullPath: 'path/group1', parent: { id: '123', }, @@ -651,7 +696,7 @@ describe('GitLabClient', () => { id: 'gid://gitlab/Group/2', name: 'group2', description: 'description2', - fullPath: 'root/group2', + fullPath: 'path/group2', parent: { id: '123', }, @@ -674,7 +719,25 @@ describe('GitLabClient', () => { const saasGroups = (await client.listSaasGroups('root')).items; + const expectedSaasGroup1 = { + id: 1, + name: 'group1', + description: 'description1', + full_path: 'path/group1', + parent_id: 123, + }; + + const expectedSaasGroup2 = { + id: 2, + name: 'group2', + description: 'description2', + full_path: 'path/group2', + parent_id: 123, + }; + expect(saasGroups.length).toEqual(2); + expect(saasGroups[0]).toEqual(expectedSaasGroup2); + expect(saasGroups[1]).toEqual(expectedSaasGroup1); }); }); @@ -683,7 +746,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: { @@ -713,7 +776,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -735,7 +798,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -757,7 +820,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((req, res, ctx) => + .query('getGroupMembers', async (req, res, ctx) => res( ctx.data({ group: { From 0b569236eb2e2f82664dd5e7d6f482735c9f8ae8 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 10:37:31 +0100 Subject: [PATCH 12/33] Switch from commitEmail to publicEmail for email attribute Signed-off-by: Stephen Barry --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 4 ++-- plugins/catalog-backend-module-gitlab/src/lib/types.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index e37d7f27dc..fa4ccf9f97 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -120,7 +120,7 @@ export class GitLabClient { user { id username - commitEmail + publicEmail name state webUrl @@ -156,7 +156,7 @@ export class GitLabClient { const formattedUserResponse = { id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), username: userItem.user.username, - email: userItem.user.commitEmail, + email: userItem.user.publicEmail, name: userItem.user.name, state: userItem.user.state, web_url: userItem.user.webUrl, diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index a00c66b4a7..651fbb00a6 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -80,7 +80,7 @@ export type GitLabSaasUsersResponse = { user: { id: string; username: string; - commitEmail: string; + publicEmail: string; name: string; state: string; webUrl: string; From 74f7fa6dce4a6d3378df9a7ba900de16f79fe61b Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 10:38:59 +0100 Subject: [PATCH 13/33] Update tests after switch to publicEmail and add additional test data to SaaS full mutation test Signed-off-by: Stephen Barry --- .../src/lib/client.test.ts | 6 +- .../GitlabOrgDiscoveryEntityProvider.test.ts | 87 ++++++++++++++++--- 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index b9fdf905ee..a62a3874d5 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -413,7 +413,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - commitEmail: 'user1@example.com', + publicEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', @@ -511,7 +511,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - commitEmail: 'user1@example.com', + publicEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', @@ -524,7 +524,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/2', username: 'user2', - commitEmail: 'user2@example.com', + publicEmail: 'user2@example.com', name: 'user2', state: 'active', webUrl: 'user2.com', diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index b6c62be5a2..7339dd4a29 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -558,12 +558,23 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { nodes: [ { user: { - id: 'gid://gitlab/User/1234', - username: 'testuser', - commitEmail: 'testuser@example.com', + id: 'gid://gitlab/User/12', + username: 'testuser1', + publicEmail: 'testuser1@example.com', state: 'active', - name: 'Test User', - webUrl: 'https://gitlab.com/testuser', + name: 'Test User 1', + webUrl: 'https://gitlab.com/testuser1', + avatarUrl: 'https://secure.gravatar.com/', + }, + }, + { + user: { + id: 'gid://gitlab/User/34', + username: 'testuser2', + publicEmail: 'testuser2@example.com', + state: 'active', + name: 'Test User 2', + webUrl: 'https://gitlab.com/testuser2', avatarUrl: 'https://secure.gravatar.com/', }, }, @@ -586,8 +597,8 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { groupMembers: { nodes: req.variables.group === 'group1/group2' - ? [{ user: { id: 'gid://gitlab/User/1234' } }] - : [], + ? [{ user: { id: 'gid://gitlab/User/12' } }] + : [{ user: { id: 'gid://gitlab/User/34' } }], pageInfo: { endCursor: 'end', hasNextPage: false, @@ -615,18 +626,43 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://gitlab.com/testuser', + 'url:https://gitlab.com/testuser1', 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/testuser', - 'gitlab.com/user-login': 'https://gitlab.com/testuser', + 'url:https://gitlab.com/testuser1', + 'gitlab.com/user-login': 'https://gitlab.com/testuser1', }, - name: 'testuser', + name: 'testuser1', }, spec: { memberOf: ['group2'], profile: { - displayName: 'Test User', - email: 'testuser@example.com', + displayName: 'Test User 1', + email: 'testuser1@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser2', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser2', + 'gitlab.com/user-login': 'https://gitlab.com/testuser2', + }, + name: 'testuser2', + }, + spec: { + memberOf: ['group3'], + profile: { + displayName: 'Test User 2', + email: 'testuser2@example.com', picture: 'https://secure.gravatar.com/', }, }, @@ -658,6 +694,31 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { }, locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/group1/group3', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/group1/group3', + 'gitlab.com/team-path': 'group1/group3', + }, + description: 'Group3', + name: 'group3', + }, + spec: { + children: [], + profile: { + displayName: 'group3', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, ]; expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); From 017e9e22eef53b0ca7dc36781e5954d3b9c6071e Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 10:39:41 +0100 Subject: [PATCH 14/33] Fetch users from root SaaS group Signed-off-by: Stephen Barry --- .../src/providers/GitlabOrgDiscoveryEntityProvider.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 94d3e64a02..2b614d69c9 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -190,7 +190,8 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); } else { groups = (await client.listSaasGroups(this.config.group)).items; - users = (await client.listSaasUsers(this.config.group)).items; + users = (await client.listSaasUsers(this.config.group.split('/')[0])) + .items; } const idMappedUser: { [userId: number]: GitLabUser } = {}; From 86e2f78a12a90a4500e9e88c36ae635f16a3b23d Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 11:32:06 +0100 Subject: [PATCH 15/33] Update Gitlab Org Data doc Signed-off-by: Stephen Barry --- docs/integrations/gitlab/org.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 6c154938ee..7090dab26e 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -58,7 +58,7 @@ amount of data, this can take significant time and resources. The token used must have the `read_api` scope, and the Users and Groups fetched will be those visible to the account which provisioned the token. -**NOTE**: If any groups that are being ingested are empty groups and the account which provisioned the token is shared at a higher level group via group sharing and you don't see the expected number of Groups in the catalog you may be hitting this [GitLab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). +**NOTE**: If any groups that are being ingested are empty groups and the user which provisioned the token is shared with a higher level group via [group sharing](https://docs.gitlab.com/ee/user/group/manage.html#share-a-group-with-another-group) and you don't see the expected number of `Group` entities in the catalog you may be hitting this [Gitlab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). ```yaml catalog: From 3d73bafd85c9423948e7c8fca6a86dc37ed78019 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 11:42:52 +0100 Subject: [PATCH 16/33] Add changeset Signed-off-by: Stephen Barry --- .changeset/thick-pets-drive.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .changeset/thick-pets-drive.md diff --git a/.changeset/thick-pets-drive.md b/.changeset/thick-pets-drive.md new file mode 100644 index 0000000000..a4d8a35029 --- /dev/null +++ b/.changeset/thick-pets-drive.md @@ -0,0 +1,17 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Fix Gitlab.com user ingestion by scoping GitlabOrgDiscoveryEntityProvider to a group. + +**BREAKING** The `group` parameter is now required Gitlab.com Org Data integrations and the backend will fail to start without this option configured. + +```diff +catalog: + providers: + gitlab: + yourProviderId: + host: gitlab.com + orgEnabled: true ++ group: org/teams +``` From fe55b95b133d062857c0f8cc219ce853aa8b4437 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 15 Aug 2023 12:13:52 +0100 Subject: [PATCH 17/33] Remove GitlabSaasUsersResponse and merge with GitlabGroupMemberResponse with additional fields Signed-off-by: Stephen Barry --- .../src/lib/types.ts | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 09a9fb92c3..1c08a0e253 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -60,34 +60,17 @@ export type GitLabGroupMembersResponse = { data: { group: { groupMembers: { - nodes: { user: { id: string } }[]; - pageInfo: { - endCursor: string; - hasNextPage: boolean; - }; - }; - }; - }; -}; - -export type GitLabSaasUsersResponse = { - errors: { message: string }[]; - data: { - group: { - groupMembers: { - nodes: [ - { - user: { - id: string; - username: string; - publicEmail: string; - name: string; - state: string; - webUrl: string; - avatarUrl: string; - }; - }, - ]; + nodes: { + user: { + id: string; + username: string; + publicEmail: string; + name: string; + state: string; + webUrl: string; + avatarUrl: string; + }; + }[]; pageInfo: { endCursor: string; hasNextPage: boolean; From 9e5f59fdd2bc5abc8d2f1086f7038ff0e2ba9207 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 15 Aug 2023 12:16:28 +0100 Subject: [PATCH 18/33] Rename listSaasGroups -> listDescendantGroups Signed-off-by: Stephen Barry --- .../src/lib/client.test.ts | 18 +++++++++--------- .../src/lib/client.ts | 9 +++++---- .../GitlabOrgDiscoveryEntityProvider.test.ts | 2 +- .../GitlabOrgDiscoveryEntityProvider.ts | 2 +- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index a62a3874d5..79dd79dffa 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -575,12 +575,12 @@ describe('GitLabClient', () => { }); }); - describe('listSaasGroups', () => { + describe('listDescendantGroups', () => { it('gets all groups under root', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasGroups', async (_, res, ctx) => + .query('listDescendantGroups', async (_, res, ctx) => res( ctx.data({ group: { @@ -611,7 +611,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasGroups = (await client.listSaasGroups('group1')).items; + const saasGroups = (await client.listDescendantGroups('group1')).items; const expectedSaasGroup = [ { @@ -631,7 +631,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasGroups', async (_, res, ctx) => + .query('listDescendantGroups', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -644,7 +644,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasGroups = (await client.listSaasGroups('group1')).items; + const saasGroups = (await client.listDescendantGroups('group1')).items; expect(saasGroups).toEqual([]); }); @@ -653,7 +653,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasGroups', async (_, res, ctx) => + .query('listDescendantGroups', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -666,7 +666,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - await expect(() => client.listSaasGroups('group1')).rejects.toThrow( + await expect(() => client.listDescendantGroups('group1')).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); @@ -674,7 +674,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasGroups', async (req, res, ctx) => + .query('listDescendantGroups', async (req, res, ctx) => res( ctx.data({ group: { @@ -717,7 +717,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasGroups = (await client.listSaasGroups('root')).items; + const saasGroups = (await client.listDescendantGroups('root')).items; const expectedSaasGroup1 = { id: 1, diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index fa4ccf9f97..d49811b714 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -22,7 +22,6 @@ import { import { Logger } from 'winston'; import { GitLabGroup, - GitLabSaasUsersResponse, GitLabSaasGroupsResponse, GitLabGroupMembersResponse, GitLabUser, @@ -103,7 +102,7 @@ export class GitLabClient { let endCursor: string | null = null; do { - const response: GitLabSaasUsersResponse = await fetch( + const response: GitLabGroupMembersResponse = await fetch( `${this.config.baseUrl}/api/graphql`, { method: 'POST', @@ -170,7 +169,9 @@ export class GitLabClient { return { items }; } - async listSaasGroups(groupPath: string): Promise> { + async listDescendantGroups( + groupPath: string, + ): Promise> { const items: GitLabGroup[] = []; let hasNextPage: boolean = false; let endCursor: string | null = null; @@ -186,7 +187,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query listSaasGroups($group: ID!, $endCursor: String) { + query: `query listDescendantGroups($group: ID!, $endCursor: String) { group(fullPath: $group) { descendantGroups(first: 100, after: $endCursor){ nodes{ diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 7339dd4a29..0a1284b22c 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -514,7 +514,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { server.use( graphql .link('https://gitlab.com/api/graphql') - .query('listSaasGroups', async (_, res, ctx) => + .query('listDescendantGroups', async (_, res, ctx) => res( ctx.data({ group: { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 2b614d69c9..a14a00d718 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -189,7 +189,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { active: true, }); } else { - groups = (await client.listSaasGroups(this.config.group)).items; + groups = (await client.listDescendantGroups(this.config.group)).items; users = (await client.listSaasUsers(this.config.group.split('/')[0])) .items; } From b9578fd485daade2723444ea4eb067327fb1d691 Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:10:03 +0100 Subject: [PATCH 19/33] Update .changeset/thick-pets-drive.md Co-authored-by: Jamie Klassen Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- .changeset/thick-pets-drive.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/thick-pets-drive.md b/.changeset/thick-pets-drive.md index a4d8a35029..ee09f8af06 100644 --- a/.changeset/thick-pets-drive.md +++ b/.changeset/thick-pets-drive.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend-module-gitlab': patch +'@backstage/plugin-catalog-backend-module-gitlab': minor --- Fix Gitlab.com user ingestion by scoping GitlabOrgDiscoveryEntityProvider to a group. From 5a3842761c2d6f0964d754d99ff231b0ed589bd4 Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:10:28 +0100 Subject: [PATCH 20/33] Update plugins/catalog-backend-module-gitlab/src/lib/client.ts Co-authored-by: Jamie Klassen Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index fa4ccf9f97..dca545eab3 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -118,13 +118,13 @@ export class GitLabClient { groupMembers(first: 100, relations: [DESCENDANTS], after: $endCursor) { nodes { user { - id - username + id + username publicEmail name state webUrl - avatarUrl + avatarUrl } } pageInfo { From ecf970e4fe8479d475740a2517e1ba8e7adf720b Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:10:47 +0100 Subject: [PATCH 21/33] Update plugins/catalog-backend-module-gitlab/src/lib/client.ts Co-authored-by: Jamie Klassen Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- .../src/lib/client.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index dca545eab3..f44aec9b20 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -187,24 +187,24 @@ export class GitLabClient { body: JSON.stringify({ variables: { group: groupPath, endCursor }, query: `query listSaasGroups($group: ID!, $endCursor: String) { - group(fullPath: $group) { - descendantGroups(first: 100, after: $endCursor){ - nodes{ - id - name - description - fullPath - parent{ - id - } - } - pageInfo { - endCursor - hasNextPage + group(fullPath: $group) { + descendantGroups(first: 100, after: $endCursor) { + nodes { + id + name + description + fullPath + parent { + id } + } + pageInfo { + endCursor + hasNextPage + } } - } - }`, + } + }`, }), }, ).then(r => r.json()); From 31d50133b4bc80fe07c8064d3fa58fcb16222347 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Wed, 16 Aug 2023 10:15:12 +0100 Subject: [PATCH 22/33] Remove listSaasUsers method and switch to paramterized getGroupMembers passing relations and update tests Signed-off-by: Stephen Barry --- .../src/lib/client.test.ts | 67 +++++++--- .../src/lib/client.ts | 116 +++++------------- .../GitlabOrgDiscoveryEntityProvider.test.ts | 6 +- .../GitlabOrgDiscoveryEntityProvider.ts | 17 ++- 4 files changed, 97 insertions(+), 109 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 79dd79dffa..28c5eb2775 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -398,12 +398,12 @@ describe('GitLabClient', () => { ]); }); - describe('listSaasUsers', () => { + describe('get gitlab.com users', () => { it('gets all users under group', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasUsers', async (_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: { @@ -436,7 +436,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasMembers = (await client.listSaasUsers('group1')).items; + const saasMembers = ( + await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + ).items; const expectedSaasMember = [ { id: 1, @@ -457,7 +459,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasUsers', async (_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -470,7 +472,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasMembers = (await client.listSaasUsers('group1')).items; + const saasMembers = ( + await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + ).items; expect(saasMembers).toEqual([]); }); @@ -479,7 +483,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasUsers', async (_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -492,7 +496,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - await expect(() => client.listSaasUsers('group1')).rejects.toThrow( + await expect(() => + client.getGroupMembers('group1', 'DIRECT, DESCENDANTS'), + ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); @@ -500,7 +506,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasUsers', async (req, res, ctx) => + .query('getGroupMembers', async (req, res, ctx) => res( ctx.data({ group: { @@ -547,7 +553,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasMembers = (await client.listSaasUsers('group1')).items; + const saasMembers = ( + await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + ).items; const expectedSaasMember1 = { id: 1, @@ -751,7 +759,19 @@ describe('GitLabClient', () => { ctx.data({ group: { groupMembers: { - nodes: [{ user: { id: 'gid://gitlab/User/1' } }], + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ], pageInfo: { endCursor: 'end', hasNextPage: false, @@ -767,9 +787,19 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1'); + const members = await client.getGroupMembers('group1', 'DIRECT'); - expect(members).toEqual([1]); + const user = { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }; + + expect(members.items).toEqual([user]); }); it('gets member IDs with token without full permissions', async () => { @@ -789,9 +819,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1'); + const members = await client.getGroupMembers('group1', 'DIRECT'); - expect(members).toEqual([]); + expect(members.items).toEqual([]); }); it('rejects when GraphQL returns errors', async () => { @@ -811,7 +841,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - await expect(() => client.getGroupMembers('group1')).rejects.toThrow( + await expect(() => + client.getGroupMembers('group1', 'DIRECT'), + ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); @@ -843,9 +875,10 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1'); + const members = await client.getGroupMembers('group1', 'DIRECT'); - expect(members).toEqual([1, 2]); + expect(members.items[0].id).toEqual(1); + expect(members.items[1].id).toEqual(2); }); }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index dee9f4ec51..5eee68cbe2 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -96,79 +96,6 @@ export class GitLabClient { return this.pagedRequest(`/groups`, options); } - async listSaasUsers(groupPath: string): Promise> { - const items: GitLabUser[] = []; - let hasNextPage: boolean = false; - let endCursor: string | null = null; - - do { - const response: GitLabGroupMembersResponse = await fetch( - `${this.config.baseUrl}/api/graphql`, - { - method: 'POST', - headers: { - ...getGitLabRequestOptions(this.config).headers, - ['Content-Type']: 'application/json', - }, - body: JSON.stringify({ - variables: { group: groupPath, endCursor }, - query: `query listSaasUsers($group: ID!, $endCursor: String) { - group(fullPath: $group) { - groupMembers(first: 100, relations: [DESCENDANTS], after: $endCursor) { - nodes { - user { - id - username - publicEmail - name - state - webUrl - avatarUrl - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - }`, - }), - }, - ).then(r => r.json()); - if (response.errors) { - throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); - } - - if (!response.data.group?.groupMembers?.nodes) { - this.logger.warn( - `Couldn't get members under ${groupPath}. The provided token might not have sufficient permissions`, - ); - continue; - } - - const usersData = response.data.group.groupMembers.nodes; - - for (let i = 0; i < usersData.length; i++) { - const userItem = usersData[i]; - - const formattedUserResponse = { - id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), - username: userItem.user.username, - email: userItem.user.publicEmail, - name: userItem.user.name, - state: userItem.user.state, - web_url: userItem.user.webUrl, - avatar_url: userItem.user.avatarUrl, - }; - - items.push(formattedUserResponse); - } - ({ hasNextPage, endCursor } = response.data.group.groupMembers.pageInfo); - } while (hasNextPage); - return { items }; - } - async listDescendantGroups( groupPath: string, ): Promise> { @@ -243,8 +170,11 @@ export class GitLabClient { return { items }; } - async getGroupMembers(groupPath: string): Promise { - const memberIds = []; + async getGroupMembers( + groupPath: string, + relations: string, + ): Promise> { + const items: GitLabUser[] = []; let hasNextPage: boolean = false; let endCursor: string | null = null; do { @@ -257,13 +187,19 @@ export class GitLabClient { ['Content-Type']: 'application/json', }, body: JSON.stringify({ - variables: { group: groupPath, endCursor }, + variables: { group: groupPath, relations: relations, endCursor }, query: `query getGroupMembers($group: ID!, $endCursor: String) { group(fullPath: $group) { - groupMembers(first: 100, relations: [DIRECT], after: $endCursor) { + groupMembers(first: 100, relations: [$relations], after: $endCursor) { nodes { user { id + username + publicEmail + name + state + webUrl + avatarUrl } } pageInfo { @@ -287,16 +223,26 @@ export class GitLabClient { continue; } - memberIds.push( - ...response.data.group.groupMembers.nodes - .filter(n => n.user) - .map(node => - Number(node.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), - ), - ); + const usersData = response.data.group.groupMembers.nodes; + + for (let i = 0; i < usersData.length; i++) { + const userItem = usersData[i]; + + const formattedUserResponse = { + id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), + username: userItem.user.username, + email: userItem.user.publicEmail, + name: userItem.user.name, + state: userItem.user.state, + web_url: userItem.user.webUrl, + avatar_url: userItem.user.avatarUrl, + }; + + items.push(formattedUserResponse); + } ({ hasNextPage, endCursor } = response.data.group.groupMembers.pageInfo); } while (hasNextPage); - return memberIds; + return { items }; } /** diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 0a1284b22c..1b1357c1bf 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -550,7 +550,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ), graphql .link('https://gitlab.com/api/graphql') - .query('listSaasUsers', async (_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: { @@ -634,7 +634,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { name: 'testuser1', }, spec: { - memberOf: ['group2'], + memberOf: ['group2', 'group3'], profile: { displayName: 'Test User 1', email: 'testuser1@example.com', @@ -659,7 +659,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { name: 'testuser2', }, spec: { - memberOf: ['group3'], + memberOf: ['group2', 'group3'], profile: { displayName: 'Test User 2', email: 'testuser2@example.com', diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index a14a00d718..bec3b06661 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -190,8 +190,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); } else { groups = (await client.listDescendantGroups(this.config.group)).items; - users = (await client.listSaasUsers(this.config.group.split('/')[0])) - .items; + users = ( + await client.getGroupMembers( + this.config.group.split('/')[0], + 'DIRECT, DESCENDANTS', + ) + ).items; } const idMappedUser: { [userId: number]: GitLabUser } = {}; @@ -236,8 +240,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupRes.scanned++; groupRes.matches.push(group); - for (const id of await client.getGroupMembers(group.full_path)) { - const user = idMappedUser[id]; + const groupUsers = await client.getGroupMembers( + group.full_path, + 'DIRECT', + ); + + for (const groupUser of groupUsers.items) { + const user = idMappedUser[groupUser.id]; if (user) { user.groups = (user.groups ?? []).concat(group); } From aef4f7263ae1bf8f13f2a025279819f6d92e3439 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Wed, 16 Aug 2023 11:41:33 +0100 Subject: [PATCH 23/33] Fix relations argument declaration for getGroupMembers query and fix relation for fetching SaaS users Signed-off-by: Stephen Barry --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 2 +- .../src/providers/GitlabOrgDiscoveryEntityProvider.ts | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 5eee68cbe2..52829c343b 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -188,7 +188,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, - query: `query getGroupMembers($group: ID!, $endCursor: String) { + query: `query getGroupMembers($group: ID!, $relations: GroupMemberRelation!, $endCursor: String) { group(fullPath: $group) { groupMembers(first: 100, relations: [$relations], after: $endCursor) { nodes { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index bec3b06661..5c4624c43a 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -191,10 +191,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { } else { groups = (await client.listDescendantGroups(this.config.group)).items; users = ( - await client.getGroupMembers( - this.config.group.split('/')[0], - 'DIRECT, DESCENDANTS', - ) + await client.getGroupMembers(this.config.group.split('/')[0], 'DIRECT') ).items; } From cb61a63af3d8d14c65501c54f2ce49a763a6d36c Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Wed, 16 Aug 2023 14:43:25 +0100 Subject: [PATCH 24/33] Switch relations argument to string[] and update references Signed-off-by: Stephen Barry --- .../src/lib/client.test.ts | 16 ++++++++-------- .../src/lib/client.ts | 6 +++--- .../GitlabOrgDiscoveryEntityProvider.ts | 10 ++++++---- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 28c5eb2775..2bb1f713b5 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -437,7 +437,7 @@ describe('GitLabClient', () => { }); const saasMembers = ( - await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) ).items; const expectedSaasMember = [ { @@ -473,7 +473,7 @@ describe('GitLabClient', () => { }); const saasMembers = ( - await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) ).items; expect(saasMembers).toEqual([]); @@ -497,7 +497,7 @@ describe('GitLabClient', () => { }); await expect(() => - client.getGroupMembers('group1', 'DIRECT, DESCENDANTS'), + client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); @@ -554,7 +554,7 @@ describe('GitLabClient', () => { }); const saasMembers = ( - await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) ).items; const expectedSaasMember1 = { @@ -787,7 +787,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', 'DIRECT'); + const members = await client.getGroupMembers('group1', ['DIRECT']); const user = { id: 1, @@ -819,7 +819,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', 'DIRECT'); + const members = await client.getGroupMembers('group1', ['DIRECT']); expect(members.items).toEqual([]); }); @@ -842,7 +842,7 @@ describe('GitLabClient', () => { }); await expect(() => - client.getGroupMembers('group1', 'DIRECT'), + client.getGroupMembers('group1', ['DIRECT']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); @@ -875,7 +875,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', 'DIRECT'); + const members = await client.getGroupMembers('group1', ['DIRECT']); expect(members.items[0].id).toEqual(1); expect(members.items[1].id).toEqual(2); diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 006a345c4a..e2c4914979 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -173,7 +173,7 @@ export class GitLabClient { async getGroupMembers( groupPath: string, - relations: string, + relations: string[], ): Promise> { const items: GitLabUser[] = []; let hasNextPage: boolean = false; @@ -189,9 +189,9 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, - query: `query getGroupMembers($group: ID!, $relations: GroupMemberRelation!, $endCursor: String) { + query: `query getGroupMembers($group: ID!, $relations: [GroupMemberRelation!], $endCursor: String) { group(fullPath: $group) { - groupMembers(first: 100, relations: [$relations], after: $endCursor) { + groupMembers(first: 100, relations: $relations, after: $endCursor) { nodes { user { id diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 5c4624c43a..0ef0e2b23c 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -191,7 +191,10 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { } else { groups = (await client.listDescendantGroups(this.config.group)).items; users = ( - await client.getGroupMembers(this.config.group.split('/')[0], 'DIRECT') + await client.getGroupMembers(this.config.group.split('/')[0], [ + 'DIRECT', + 'DESCENDANTS', + ]) ).items; } @@ -237,10 +240,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupRes.scanned++; groupRes.matches.push(group); - const groupUsers = await client.getGroupMembers( - group.full_path, + const groupUsers = await client.getGroupMembers(group.full_path, [ 'DIRECT', - ); + ]); for (const groupUser of groupUsers.items) { const user = idMappedUser[groupUser.id]; From 1825c689d0605a295d603d182269f9e216db9567 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Wed, 16 Aug 2023 14:45:00 +0100 Subject: [PATCH 25/33] Update GitLab Org Data docs Signed-off-by: Stephen Barry --- docs/integrations/gitlab/org.md | 45 ++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 7090dab26e..5078e93d77 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -58,8 +58,6 @@ amount of data, this can take significant time and resources. The token used must have the `read_api` scope, and the Users and Groups fetched will be those visible to the account which provisioned the token. -**NOTE**: If any groups that are being ingested are empty groups and the user which provisioned the token is shared with a higher level group via [group sharing](https://docs.gitlab.com/ee/user/group/manage.html#share-a-group-with-another-group) and you don't see the expected number of `Group` entities in the catalog you may be hitting this [Gitlab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). - ```yaml catalog: providers: @@ -67,11 +65,52 @@ catalog: yourProviderId: host: gitlab.com orgEnabled: true - group: org/teams # Required for gitlab.com. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) + group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything ``` +### Groups + When the `group` parameter is provided, the corresponding path prefix will be stripped out from each matching group when computing the unique entity name. e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will be `avengers-gotg`. + +For gitlab.com, when `orgEnabled: true`, the `group` parameter is required in +order to limit the ingestion to a group within your organisation. `Group` +entities will only be ingested for the configured group, or it's descendant groups, +but not any ancestor groups higher than the configured group path. Only groups +which contain members will be ingested. + +### Users + +For self hosted, all `User` entities are ingested from the entire instance. + +For gitlab.com `User` entities for users who have [direct or inherited membership](https://docs.gitlab.com/ee/user/project/members/index.html#membership-types) +of the top-level group for the configured group path will be ingested. + +### Limiting `User` and `Group` entity ingestion in the provider + +Optionally, you can limit the entity types ingested by the provider when using +`orgEnabled: true` with the following `rules` configuration to limit it to only +`User` and `Group` entities. + +```yaml +catalog: + providers: + gitlab: + yourOrgDataProviderId: + host: gitlab.com + orgEnabled: true + group: org/teams + rules: + - allow: [Group, User] +``` + +## Troubleshooting + +**NOTE**: If any groups that are being ingested are empty groups (i.e. do not +contain any projects) and the user which provisioned the token is shared with a +higher level group via [group sharing](https://docs.gitlab.com/ee/user/group/manage.html#share-a-group-with-another-group) +and you don't see the expected number of `Group` entities in the catalog you may +be hitting this [Gitlab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). From 8bc51413d9069cf6cc50ac6c7f57adbaf658505a Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 22 Aug 2023 08:26:01 +0100 Subject: [PATCH 26/33] Apply suggestions from code review Co-authored-by: Jamie Klassen Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- docs/integrations/gitlab/org.md | 2 +- .../src/lib/client.ts | 34 ++++++++++--------- .../src/lib/types.ts | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 5078e93d77..ca40cefa9e 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -78,7 +78,7 @@ e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will be For gitlab.com, when `orgEnabled: true`, the `group` parameter is required in order to limit the ingestion to a group within your organisation. `Group` -entities will only be ingested for the configured group, or it's descendant groups, +entities will only be ingested for the configured group, or its descendant groups, but not any ancestor groups higher than the configured group path. Only groups which contain members will be ingested. diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index e2c4914979..3675d7e779 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -115,25 +115,27 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query listDescendantGroups($group: ID!, $endCursor: String) { - group(fullPath: $group) { - descendantGroups(first: 100, after: $endCursor) { - nodes { - id - name - description - fullPath - parent { + query: /* GraphQL */ ` + query listDescendantGroups($group: ID!, $endCursor: String) { + group(fullPath: $group) { + descendantGroups(first: 100, after: $endCursor) { + nodes { id + name + description + fullPath + parent { + id + } + } + pageInfo { + endCursor + hasNextPage } } - pageInfo { - endCursor - hasNextPage - } - } + } } - }`, + `, }), }, ).then(r => r.json()); @@ -189,7 +191,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, - query: `query getGroupMembers($group: ID!, $relations: [GroupMemberRelation!], $endCursor: String) { + query: /* GraphQL */ `query getGroupMembers($group: ID!, $relations: [GroupMemberRelation!], $endCursor: String) { group(fullPath: $group) { groupMembers(first: 100, relations: $relations, after: $endCursor) { nodes { diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 1c08a0e253..5a4082eec3 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -80,7 +80,7 @@ export type GitLabGroupMembersResponse = { }; }; -export type GitLabSaasGroupsResponse = { +export type GitLabDescendantGroupsResponse = { errors: { message: string }[]; data: { group: { From 880c08c0b0b9afed2fe555692e28e8ce5344b00b Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 22 Aug 2023 08:33:19 +0100 Subject: [PATCH 27/33] Apply prettier lint and rename GitlabSaasGroupsResponse -> GitlabDescendantGroupsResponse Signed-off-by: Stephen Barry --- .../src/lib/client.ts | 48 +++++++++++-------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 3675d7e779..681c0c7603 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -22,7 +22,7 @@ import { import { Logger } from 'winston'; import { GitLabGroup, - GitLabSaasGroupsResponse, + GitLabDescendantGroupsResponse, GitLabGroupMembersResponse, GitLabUser, } from './types'; @@ -105,7 +105,7 @@ export class GitLabClient { let endCursor: string | null = null; do { - const response: GitLabSaasGroupsResponse = await fetch( + const response: GitLabDescendantGroupsResponse = await fetch( `${this.config.baseUrl}/api/graphql`, { method: 'POST', @@ -191,27 +191,37 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, - query: /* GraphQL */ `query getGroupMembers($group: ID!, $relations: [GroupMemberRelation!], $endCursor: String) { - group(fullPath: $group) { - groupMembers(first: 100, relations: $relations, after: $endCursor) { - nodes { - user { - id - username - publicEmail - name - state - webUrl - avatarUrl + query: /* GraphQL */ ` + query getGroupMembers( + $group: ID! + $relations: [GroupMemberRelation!] + $endCursor: String + ) { + group(fullPath: $group) { + groupMembers( + first: 100 + relations: $relations + after: $endCursor + ) { + nodes { + user { + id + username + publicEmail + name + state + webUrl + avatarUrl + } + } + pageInfo { + endCursor + hasNextPage } - } - pageInfo { - endCursor - hasNextPage } } } - }`, + `, }), }, ).then(r => r.json()); From d7403140aed62e63b09630ab8c5788f3defd06a4 Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:53:13 +0100 Subject: [PATCH 28/33] Update plugins/catalog-backend-module-gitlab/src/lib/client.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply suggestion Co-authored-by: Fredrik Adelöw Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 681c0c7603..4a5d70b0bf 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -187,7 +187,7 @@ export class GitLabClient { method: 'POST', headers: { ...getGitLabRequestOptions(this.config).headers, - ['Content-Type']: 'application/json', + 'Content-Type': 'application/json', }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, From 69405c52f8d1d72726994705e41798616b426110 Mon Sep 17 00:00:00 2001 From: angom1 Date: Thu, 31 Aug 2023 12:29:12 +0100 Subject: [PATCH 29/33] Simplify for loops Signed-off-by: angom1 --- .../catalog-backend-module-gitlab/src/lib/client.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 4a5d70b0bf..65eb3f326a 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -150,11 +150,7 @@ export class GitLabClient { continue; } - const groupsData = response.data.group.descendantGroups.nodes; - - for (let i = 0; i < groupsData.length; i++) { - const groupItem = groupsData[i]; - + for (const groupItem of response.data.group.descendantGroups.nodes) { const formattedGroupResponse = { id: Number(groupItem.id.replace(/^gid:\/\/gitlab\/Group\//, '')), name: groupItem.name, @@ -236,11 +232,7 @@ export class GitLabClient { continue; } - const usersData = response.data.group.groupMembers.nodes; - - for (let i = 0; i < usersData.length; i++) { - const userItem = usersData[i]; - + for (const userItem of response.data.group.groupMembers.nodes) { const formattedUserResponse = { id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), username: userItem.user.username, From 1d8b7a51ea9a4f21ab602e67d54580f7a095e8af Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 1 Sep 2023 11:11:43 +0100 Subject: [PATCH 30/33] Switch to commitEmail Signed-off-by: angom1 --- .../catalog-backend-module-gitlab/src/lib/client.test.ts | 8 ++++---- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 2 +- plugins/catalog-backend-module-gitlab/src/lib/types.ts | 2 +- .../providers/GitlabOrgDiscoveryEntityProvider.test.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 2bb1f713b5..dda30fd86a 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -413,7 +413,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - publicEmail: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', @@ -517,7 +517,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - publicEmail: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', @@ -530,7 +530,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/2', username: 'user2', - publicEmail: 'user2@example.com', + commitEmail: 'user2@example.com', name: 'user2', state: 'active', webUrl: 'user2.com', @@ -764,7 +764,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - publicEmail: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 65eb3f326a..26d1b2a6fb 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -236,7 +236,7 @@ export class GitLabClient { const formattedUserResponse = { id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), username: userItem.user.username, - email: userItem.user.publicEmail, + email: userItem.user.commitEmail, name: userItem.user.name, state: userItem.user.state, web_url: userItem.user.webUrl, diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 5a4082eec3..a339ac57e4 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -64,7 +64,7 @@ export type GitLabGroupMembersResponse = { user: { id: string; username: string; - publicEmail: string; + commitEmail: string; name: string; state: string; webUrl: string; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 1b1357c1bf..ce00510140 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -560,7 +560,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { user: { id: 'gid://gitlab/User/12', username: 'testuser1', - publicEmail: 'testuser1@example.com', + commitEmail: 'testuser1@example.com', state: 'active', name: 'Test User 1', webUrl: 'https://gitlab.com/testuser1', @@ -571,7 +571,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { user: { id: 'gid://gitlab/User/34', username: 'testuser2', - publicEmail: 'testuser2@example.com', + commitEmail: 'testuser2@example.com', state: 'active', name: 'Test User 2', webUrl: 'https://gitlab.com/testuser2', From 739b927df9cdb11bc8ae04dbc1b16cb3334afebf Mon Sep 17 00:00:00 2001 From: angom1 Date: Tue, 5 Sep 2023 10:50:37 +0100 Subject: [PATCH 31/33] Add check for null response Signed-off-by: angom1 --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 26d1b2a6fb..cf716ce521 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -150,7 +150,9 @@ export class GitLabClient { continue; } - for (const groupItem of response.data.group.descendantGroups.nodes) { + for (const groupItem of response.data.group.descendantGroups.nodes.filter( + group => group.id, + )) { const formattedGroupResponse = { id: Number(groupItem.id.replace(/^gid:\/\/gitlab\/Group\//, '')), name: groupItem.name, @@ -232,7 +234,9 @@ export class GitLabClient { continue; } - for (const userItem of response.data.group.groupMembers.nodes) { + for (const userItem of response.data.group.groupMembers.nodes.filter( + user => user.user.id, + )) { const formattedUserResponse = { id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), username: userItem.user.username, From 0ccb00afa0dca7393b1545861c838cd3051b9192 Mon Sep 17 00:00:00 2001 From: angom1 Date: Thu, 7 Sep 2023 10:57:17 +0100 Subject: [PATCH 32/33] Update query to commitEmail Signed-off-by: angom1 --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index cf716ce521..fe0da6c4b5 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -205,7 +205,7 @@ export class GitLabClient { user { id username - publicEmail + commitEmail name state webUrl From 2c0ac5c2b4fc8f0c6f8023d4e9918d2704a7c164 Mon Sep 17 00:00:00 2001 From: angom1 Date: Tue, 12 Sep 2023 11:19:52 +0100 Subject: [PATCH 33/33] Update filter syntax Signed-off-by: angom1 --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index fe0da6c4b5..79ae328ce2 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -151,7 +151,7 @@ export class GitLabClient { } for (const groupItem of response.data.group.descendantGroups.nodes.filter( - group => group.id, + group => group?.id, )) { const formattedGroupResponse = { id: Number(groupItem.id.replace(/^gid:\/\/gitlab\/Group\//, '')), @@ -235,7 +235,7 @@ export class GitLabClient { } for (const userItem of response.data.group.groupMembers.nodes.filter( - user => user.user.id, + user => user.user?.id, )) { const formattedUserResponse = { id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')),