diff --git a/.changeset/thick-pets-drive.md b/.changeset/thick-pets-drive.md new file mode 100644 index 0000000000..ee09f8af06 --- /dev/null +++ b/.changeset/thick-pets-drive.md @@ -0,0 +1,17 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': minor +--- + +Fix Gitlab.com user ingestion by scoping GitlabOrgDiscoveryEntityProvider to a group. + +**BREAKING** The `group` parameter is now required Gitlab.com Org Data integrations and the backend will fail to start without this option configured. + +```diff +catalog: + providers: + gitlab: + yourProviderId: + host: gitlab.com + orgEnabled: true ++ group: org/teams +``` diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index d022641ab6..ca40cefa9e 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -65,11 +65,52 @@ catalog: yourProviderId: host: gitlab.com orgEnabled: true - group: org/teams # Optional. Must not end with slash. Accepts only groups under the provided path (which will be stripped) + group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything ``` +### Groups + When the `group` parameter is provided, the corresponding path prefix will be stripped out from each matching group when computing the unique entity name. e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will be `avengers-gotg`. + +For gitlab.com, when `orgEnabled: true`, the `group` parameter is required in +order to limit the ingestion to a group within your organisation. `Group` +entities will only be ingested for the configured group, or its descendant groups, +but not any ancestor groups higher than the configured group path. Only groups +which contain members will be ingested. + +### Users + +For self hosted, all `User` entities are ingested from the entire instance. + +For gitlab.com `User` entities for users who have [direct or inherited membership](https://docs.gitlab.com/ee/user/project/members/index.html#membership-types) +of the top-level group for the configured group path will be ingested. + +### Limiting `User` and `Group` entity ingestion in the provider + +Optionally, you can limit the entity types ingested by the provider when using +`orgEnabled: true` with the following `rules` configuration to limit it to only +`User` and `Group` entities. + +```yaml +catalog: + providers: + gitlab: + yourOrgDataProviderId: + host: gitlab.com + orgEnabled: true + group: org/teams + rules: + - allow: [Group, User] +``` + +## Troubleshooting + +**NOTE**: If any groups that are being ingested are empty groups (i.e. do not +contain any projects) and the user which provisioned the token is shared with a +higher level group via [group sharing](https://docs.gitlab.com/ee/user/group/manage.html#share-a-group-with-another-group) +and you don't see the expected number of `Group` entities in the catalog you may +be hitting this [Gitlab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index f4504e8937..dda30fd86a 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -398,17 +398,29 @@ describe('GitLabClient', () => { ]); }); - describe('getGroupMembers', () => { - it('gets member IDs', async () => { + describe('get gitlab.com users', () => { + it('gets all users under group', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: { groupMembers: { - nodes: [{ user: { id: 'gid://gitlab/User/1' } }], + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + commitEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ], pageInfo: { endCursor: 'end', hasNextPage: false, @@ -424,16 +436,30 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1'); + const saasMembers = ( + await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + ).items; + const expectedSaasMember = [ + { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }, + ]; - expect(members).toEqual([1]); + expect(saasMembers.length).toEqual(1); + expect(saasMembers).toEqual(expectedSaasMember); }); - it('gets member IDs with token without full permissions', async () => { + it('gets all users with token without full permissions', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -446,16 +472,18 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1'); + const saasMembers = ( + await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + ).items; - expect(members).toEqual([]); + expect(saasMembers).toEqual([]); }); it('rejects when GraphQL returns errors', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -468,7 +496,354 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - await expect(() => client.getGroupMembers('group1')).rejects.toThrow( + await expect(() => + client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']), + ).rejects.toThrow( + 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', + ); + }); + it('traverses multi-page results', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .query('getGroupMembers', async (req, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: req.variables.endCursor + ? [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + commitEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ] + : [ + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + commitEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasMembers = ( + await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + ).items; + + const expectedSaasMember1 = { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }; + + const expectedSaasMember2 = { + id: 2, + username: 'user2', + email: 'user2@example.com', + name: 'user2', + state: 'active', + web_url: 'user2.com', + avatar_url: 'user2', + }; + + expect(saasMembers.length).toEqual(2); + expect(saasMembers[0]).toEqual(expectedSaasMember2); + expect(saasMembers[1]).toEqual(expectedSaasMember1); + }); + }); + + describe('listDescendantGroups', () => { + it('gets all groups under root', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .query('listDescendantGroups', async (_, res, ctx) => + res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/1', + name: 'group1', + description: 'description1', + fullPath: 'path/group1', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasGroups = (await client.listDescendantGroups('group1')).items; + + const expectedSaasGroup = [ + { + id: 1, + name: 'group1', + description: 'description1', + full_path: 'path/group1', + parent_id: 123, + }, + ]; + + expect(saasGroups.length).toEqual(1); + expect(saasGroups).toEqual(expectedSaasGroup); + }); + + it('gets all descendant groups with token without full permissions', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .query('listDescendantGroups', async (_, res, ctx) => + res( + ctx.data({ + group: {}, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasGroups = (await client.listDescendantGroups('group1')).items; + + expect(saasGroups).toEqual([]); + }); + + it('rejects when GraphQL returns errors', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .query('listDescendantGroups', async (_, res, ctx) => + res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + await expect(() => client.listDescendantGroups('group1')).rejects.toThrow( + 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', + ); + }); + it('traverses multi-page results', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .query('listDescendantGroups', async (req, res, ctx) => + res( + ctx.data({ + group: { + descendantGroups: { + nodes: req.variables.endCursor + ? [ + { + id: 'gid://gitlab/Group/1', + name: 'group1', + description: 'description1', + fullPath: 'path/group1', + parent: { + id: '123', + }, + }, + ] + : [ + { + id: 'gid://gitlab/Group/2', + name: 'group2', + description: 'description2', + fullPath: 'path/group2', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasGroups = (await client.listDescendantGroups('root')).items; + + const expectedSaasGroup1 = { + id: 1, + name: 'group1', + description: 'description1', + full_path: 'path/group1', + parent_id: 123, + }; + + const expectedSaasGroup2 = { + id: 2, + name: 'group2', + description: 'description2', + full_path: 'path/group2', + parent_id: 123, + }; + + expect(saasGroups.length).toEqual(2); + expect(saasGroups[0]).toEqual(expectedSaasGroup2); + expect(saasGroups[1]).toEqual(expectedSaasGroup1); + }); + }); + + describe('getGroupMembers', () => { + it('gets member IDs', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .query('getGroupMembers', async (_, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + commitEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const members = await client.getGroupMembers('group1', ['DIRECT']); + + const user = { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }; + + expect(members.items).toEqual([user]); + }); + + it('gets member IDs with token without full permissions', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .query('getGroupMembers', async (_, res, ctx) => + res( + ctx.data({ + group: {}, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const members = await client.getGroupMembers('group1', ['DIRECT']); + + expect(members.items).toEqual([]); + }); + + it('rejects when GraphQL returns errors', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .query('getGroupMembers', async (_, res, ctx) => + res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + await expect(() => + client.getGroupMembers('group1', ['DIRECT']), + ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); @@ -477,7 +852,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((req, res, ctx) => + .query('getGroupMembers', async (req, res, ctx) => res( ctx.data({ group: { @@ -500,9 +875,10 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1'); + const members = await client.getGroupMembers('group1', ['DIRECT']); - expect(members).toEqual([1, 2]); + expect(members.items[0].id).toEqual(1); + expect(members.items[1].id).toEqual(2); }); }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 41b99e1c5b..79ae328ce2 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -20,7 +20,12 @@ import { GitLabIntegrationConfig, } from '@backstage/integration'; import { Logger } from 'winston'; -import { GitLabGroup, GitLabGroupMembersResponse, GitLabUser } from './types'; +import { + GitLabGroup, + GitLabDescendantGroupsResponse, + GitLabGroupMembersResponse, + GitLabUser, +} from './types'; export type CommonListOptions = { [key: string]: string | number | boolean | undefined; @@ -92,12 +97,15 @@ export class GitLabClient { return this.pagedRequest(`/groups`, options); } - async getGroupMembers(groupPath: string): Promise { - const memberIds = []; + async listDescendantGroups( + groupPath: string, + ): Promise> { + const items: GitLabGroup[] = []; let hasNextPage: boolean = false; let endCursor: string | null = null; + do { - const response: GitLabGroupMembersResponse = await fetch( + const response: GitLabDescendantGroupsResponse = await fetch( `${this.config.baseUrl}/api/graphql`, { method: 'POST', @@ -107,21 +115,111 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query($group: ID!, $endCursor: String) { - group(fullPath: $group) { - groupMembers(first: 100, relations: [DIRECT], after: $endCursor) { - nodes { - user { + query: /* GraphQL */ ` + query listDescendantGroups($group: ID!, $endCursor: String) { + group(fullPath: $group) { + descendantGroups(first: 100, after: $endCursor) { + nodes { id + name + description + fullPath + parent { + id + } + } + pageInfo { + endCursor + hasNextPage } - } - pageInfo { - endCursor - hasNextPage } } } - }`, + `, + }), + }, + ).then(r => r.json()); + if (response.errors) { + throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); + } + + if (!response.data.group?.descendantGroups?.nodes) { + this.logger.warn( + `Couldn't get groups under ${groupPath}. The provided token might not have sufficient permissions`, + ); + continue; + } + + for (const groupItem of response.data.group.descendantGroups.nodes.filter( + group => group?.id, + )) { + const formattedGroupResponse = { + id: Number(groupItem.id.replace(/^gid:\/\/gitlab\/Group\//, '')), + name: groupItem.name, + description: groupItem.description, + full_path: groupItem.fullPath, + parent_id: Number( + groupItem.parent.id.replace(/^gid:\/\/gitlab\/Group\//, ''), + ), + }; + + items.push(formattedGroupResponse); + } + ({ hasNextPage, endCursor } = + response.data.group.descendantGroups.pageInfo); + } while (hasNextPage); + return { items }; + } + + async getGroupMembers( + groupPath: string, + relations: string[], + ): Promise> { + const items: GitLabUser[] = []; + let hasNextPage: boolean = false; + let endCursor: string | null = null; + do { + const response: GitLabGroupMembersResponse = await fetch( + `${this.config.baseUrl}/api/graphql`, + { + method: 'POST', + headers: { + ...getGitLabRequestOptions(this.config).headers, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + variables: { group: groupPath, relations: relations, endCursor }, + query: /* GraphQL */ ` + query getGroupMembers( + $group: ID! + $relations: [GroupMemberRelation!] + $endCursor: String + ) { + group(fullPath: $group) { + groupMembers( + first: 100 + relations: $relations + after: $endCursor + ) { + nodes { + user { + id + username + commitEmail + name + state + webUrl + avatarUrl + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + `, }), }, ).then(r => r.json()); @@ -136,16 +234,24 @@ export class GitLabClient { continue; } - memberIds.push( - ...response.data.group.groupMembers.nodes - .filter(n => n.user) - .map(node => - Number(node.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), - ), - ); + for (const userItem of response.data.group.groupMembers.nodes.filter( + user => user.user?.id, + )) { + const formattedUserResponse = { + id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), + username: userItem.user.username, + email: userItem.user.commitEmail, + name: userItem.user.name, + state: userItem.user.state, + web_url: userItem.user.webUrl, + avatar_url: userItem.user.avatarUrl, + }; + + items.push(formattedUserResponse); + } ({ hasNextPage, endCursor } = response.data.group.groupMembers.pageInfo); } while (hasNextPage); - return memberIds; + return { items }; } /** diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 9b3a68c107..a339ac57e4 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -60,7 +60,17 @@ export type GitLabGroupMembersResponse = { data: { group: { groupMembers: { - nodes: { user: { id: string } }[]; + nodes: { + user: { + id: string; + username: string; + commitEmail: string; + name: string; + state: string; + webUrl: string; + avatarUrl: string; + }; + }[]; pageInfo: { endCursor: string; hasNextPage: boolean; @@ -70,6 +80,31 @@ export type GitLabGroupMembersResponse = { }; }; +export type GitLabDescendantGroupsResponse = { + errors: { message: string }[]; + data: { + group: { + descendantGroups: { + nodes: [ + { + id: string; + name: string; + description: string; + fullPath: string; + parent: { + id: string; + }; + }, + ]; + pageInfo: { + endCursor: string; + hasNextPage: false; + }; + }; + }; + }; +}; + export type GitlabProviderConfig = { host: string; group: string; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index e29cdccc04..ce00510140 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -475,6 +475,259 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { }); }); + it('apply full update on scheduled execution for gitlab.com', async () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + group: 'group1', + orgEnabled: true, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + server.use( + graphql + .link('https://gitlab.com/api/graphql') + .query('listDescendantGroups', async (_, res, ctx) => + res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/456', + name: 'group2', + description: 'Group2', + fullPath: 'group1/group2', + parent: { + id: 'gid://gitlab/Group/123', + }, + }, + { + id: 'gid://gitlab/Group/789', + name: 'group3', + description: 'Group3', + fullPath: 'group1/group3', + parent: { + id: 'gid://gitlab/Group/123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + graphql + .link('https://gitlab.com/api/graphql') + .query('getGroupMembers', async (_, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/12', + username: 'testuser1', + commitEmail: 'testuser1@example.com', + state: 'active', + name: 'Test User 1', + webUrl: 'https://gitlab.com/testuser1', + avatarUrl: 'https://secure.gravatar.com/', + }, + }, + { + user: { + id: 'gid://gitlab/User/34', + username: 'testuser2', + commitEmail: 'testuser2@example.com', + state: 'active', + name: 'Test User 2', + webUrl: 'https://gitlab.com/testuser2', + avatarUrl: 'https://secure.gravatar.com/', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + graphql + .link('https://gitlab.com/api/graphql') + .query('getGroupMembers', async (req, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: + req.variables.group === 'group1/group2' + ? [{ user: { id: 'gid://gitlab/User/12' } }] + : [{ user: { id: 'gid://gitlab/User/34' } }], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', + ); + await (taskDef.fn as () => Promise)(); + + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser1', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser1', + 'gitlab.com/user-login': 'https://gitlab.com/testuser1', + }, + name: 'testuser1', + }, + spec: { + memberOf: ['group2', 'group3'], + profile: { + displayName: 'Test User 1', + email: 'testuser1@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser2', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser2', + 'gitlab.com/user-login': 'https://gitlab.com/testuser2', + }, + name: 'testuser2', + }, + spec: { + memberOf: ['group2', 'group3'], + profile: { + displayName: 'Test User 2', + email: 'testuser2@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/group1/group2', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/group1/group2', + 'gitlab.com/team-path': 'group1/group2', + }, + description: 'Group2', + name: 'group2', + }, + spec: { + children: [], + profile: { + displayName: 'group2', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/group1/group3', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/group1/group3', + 'gitlab.com/team-path': 'group1/group3', + }, + description: 'Group3', + name: 'group3', + }, + spec: { + children: [], + profile: { + displayName: 'group3', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); + it('fail without schedule and scheduler', () => { const config = new ConfigReader({ integrations: { @@ -543,6 +796,42 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); }); + it('fail with scheduler but no group config when host is gitlab.com', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + orgEnabled: true, + }, + }, + }, + }, + }); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`, + ); + }); + it('single simple provider config with schedule in config', async () => { const schedule = new PersistingTaskRunner(); const scheduler = { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 02c036f27a..0ef0e2b23c 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -89,6 +89,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ); } + if (!providerConfig.group && providerConfig.host === 'gitlab.com') { + throw new Error( + `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`, + ); + } + if (!options.schedule && !providerConfig.schedule) { throw new Error( `No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`, @@ -168,19 +174,29 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { logger: logger, }); - const users = paginated(options => client.listUsers(options), { - page: 1, - per_page: 100, - active: true, - }); + let groups; + let users; - const groups = paginated( - options => client.listGroups(options), - { + if (client.isSelfManaged()) { + groups = paginated(options => client.listGroups(options), { page: 1, per_page: 100, - }, - ); + }); + + users = paginated(options => client.listUsers(options), { + page: 1, + per_page: 100, + active: true, + }); + } else { + groups = (await client.listDescendantGroups(this.config.group)).items; + users = ( + await client.getGroupMembers(this.config.group.split('/')[0], [ + 'DIRECT', + 'DESCENDANTS', + ]) + ).items; + } const idMappedUser: { [userId: number]: GitLabUser } = {}; @@ -224,8 +240,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupRes.scanned++; groupRes.matches.push(group); - for (const id of await client.getGroupMembers(group.full_path)) { - const user = idMappedUser[id]; + const groupUsers = await client.getGroupMembers(group.full_path, [ + 'DIRECT', + ]); + + for (const groupUser of groupUsers.items) { + const user = idMappedUser[groupUser.id]; if (user) { user.groups = (user.groups ?? []).concat(group); }