paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 42 Packages Projects Releases Wiki Activity

Merge pull request #16856 from jamieklassen/gitlab-orgdata-nonadmin

Browse Source 
Gitlab Org Data integration fetches memberships via graphQL
This commit is contained in:
Fredrik Adelöw
2023-03-15 13:39:49 +01:00
committed by GitHub
parent e63a7fe9bb 67892fd4d9
commit e00f38260f
7 changed files with 350 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/hungry-lies-cry.md
+8
View File
@@ -0,0 +1,8 @@
---
'@backstage/plugin-catalog-backend-module-gitlab': patch
---
The gitlab org data integration now makes use of the GraphQL API to determine
the relationships between imported User and Group entities, effectively making
this integration usable without an administrator account's Personal Access
Token.
docs/integrations/gitlab/org.md
+13 -12
View File
@@ -2,15 +2,14 @@
id: org
title: GitLab Organizational Data
sidebar_label: Org Data
description: Importing users and groups from a GitLab organization into Backstage
description: Importing users and groups from GitLab into Backstage
---
The Backstage catalog can be set up to ingest organizational data - users and
teams - directly from an organization in GitLab. The result
is a hierarchy of
The Backstage catalog can be set up to ingest organizational data -- users and
groups -- directly from GitLab. The result is a hierarchy of
[`User`](../../features/software-catalog/descriptor-format.md#kind-user) and
[`Group`](../../features/software-catalog/descriptor-format.md#kind-group) kind
entities that mirror your org setup.
[`Group`](../../features/software-catalog/descriptor-format.md#kind-group)
entities that mirrors your org setup.
```yaml
integrations:
@@ -19,10 +18,11 @@ integrations:
token: ${GITLAB_TOKEN}
```
This will query all users and groups from your gitlab installation. Depending on the size
of the Gitlab Instance, this can take some time and resources.
This will query all users and groups from your GitLab instance. Depending on the
amount of data, this can take significant time and resources.
The token that is used for the Organization Integration, has to be an Admin Personal Access Token (PAT).
The token used must have the `read_api` scope, and the Users and Groups fetched
will be those visible to the account which provisioned the token.
```yaml
catalog:
@@ -35,6 +35,7 @@ catalog:
groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything
```
When the `group` parameter is provided, the corresponding path prefix will be stripped out from each matching group
when computing the unique entity name. e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will
be `avengers-gotg`.
When the `group` parameter is provided, the corresponding path prefix will be
stripped out from each matching group when computing the unique entity name.
e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will be
`avengers-gotg`.
plugins/catalog-backend-module-gitlab/src/lib/client.test.ts
+225 -1
View File
@@ -18,9 +18,10 @@ import { ConfigReader } from '@backstage/config';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import { readGitLabIntegrationConfig } from '@backstage/integration';
import { getVoidLogger } from '@backstage/backend-common';
import { rest } from 'msw';
import { graphql, rest } from 'msw';
import { setupServer, SetupServerApi } from 'msw/node';
import { GitLabClient, paginated } from './client';
import { GitLabGroup, GitLabUser } from './types';
const server = setupServer();
setupRequestMockHandlers(server);
@@ -30,6 +31,7 @@ const MOCK_CONFIG = readGitLabIntegrationConfig(
host: 'example.com',
token: 'test-token',
apiBaseUrl: 'https://example.com/api/v4',
baseUrl: 'https://example.com',
}),
);
const FAKE_PAGED_ENDPOINT = `/some-endpoint`;
@@ -259,6 +261,228 @@ describe('GitLabClient', () => {
expect(allProjects).toHaveLength(2);
});
});
it('listUsers gets all users in the instance', async () => {
server.use(
rest.get(`${MOCK_CONFIG.apiBaseUrl}/users`, (_, res, ctx) =>
res(
ctx.set('x-next-page', ''),
ctx.json([
{
id: 1,
username: 'test1',
name: 'Test Testit',
state: 'active',
avatar_url: 'https://secure.gravatar.com/',
web_url: 'https://gitlab.example/test1',
created_at: '2023-01-19T07:27:03.333Z',
bio: '',
location: null,
public_email: null,
skype: '',
linkedin: '',
twitter: '',
website_url: '',
organization: null,
job_title: '',
pronouns: null,
bot: false,
work_information: null,
followers: 0,
following: 0,
is_followed: false,
local_time: null,
last_sign_in_at: '2023-01-19T07:27:49.601Z',
confirmed_at: '2023-01-19T07:27:02.905Z',
last_activity_on: '2023-01-19',
email: 'test@example.com',
theme_id: 1,
color_scheme_id: 1,
projects_limit: 100000,
current_sign_in_at: '2023-01-19T09:09:10.676Z',
identities: [],
can_create_group: true,
can_create_project: true,
two_factor_enabled: false,
external: false,
private_profile: false,
commit_email: 'test@example.com',
is_admin: false,
note: '',
},
]),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const allUsers: GitLabUser[] = [];
for await (const user of paginated(
options => client.listUsers(options),
{},
)) {
allUsers.push(user);
}
expect(allUsers).toMatchObject([
{
id: 1,
username: 'test1',
email: 'test@example.com',
name: 'Test Testit',
state: 'active',
web_url: 'https://gitlab.example/test1',
avatar_url: 'https://secure.gravatar.com/',
},
]);
});
it('listGroups gets all groups in the instance', async () => {
server.use(
rest.get(`${MOCK_CONFIG.apiBaseUrl}/groups`, (_, res, ctx) =>
res(
ctx.set('x-next-page', ''),
ctx.json([
{
id: 1,
web_url: 'https://gitlab.example/groups/group1',
name: 'group1',
path: 'group1',
description: '',
visibility: 'internal',
share_with_group_lock: false,
require_two_factor_authentication: false,
two_factor_grace_period: 48,
project_creation_level: 'developer',
auto_devops_enabled: null,
subgroup_creation_level: 'owner',
emails_disabled: null,
mentions_disabled: null,
lfs_enabled: true,
default_branch_protection: 2,
avatar_url: null,
request_access_enabled: false,
full_name: '8020',
full_path: '8020',
created_at: '2017-06-19T06:42:34.160Z',
parent_id: null,
},
]),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const allGroups: GitLabGroup[] = [];
for await (const group of paginated(
options => client.listGroups(options),
{},
)) {
allGroups.push(group);
}
expect(allGroups).toMatchObject([
{
id: 1,
name: 'group1',
full_path: '8020',
description: '',
parent_id: null,
},
]);
});
describe('getGroupMembers', () => {
it('gets member IDs', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.operation((_, res, ctx) =>
res(
ctx.data({
group: {
groupMembers: {
nodes: [{ user: { id: 'gid://gitlab/User/1' } }],
pageInfo: {
endCursor: 'end',
hasNextPage: false,
},
},
},
}),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const members = await client.getGroupMembers('group1');
expect(members).toEqual([1]);
});
it('rejects when GraphQL returns errors', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.operation((_, res, ctx) =>
res(
ctx.errors([
{ message: 'Unexpected end of document', locations: [] },
]),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
await expect(() => client.getGroupMembers('group1')).rejects.toThrow(
'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]',
);
});
it('traverses multi-page results', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.operation((req, res, ctx) =>
res(
ctx.data({
group: {
groupMembers: {
nodes: req.variables.endCursor
? [{ user: { id: 'gid://gitlab/User/2' } }]
: [{ user: { id: 'gid://gitlab/User/1' } }],
pageInfo: {
endCursor: req.variables.endCursor ? 'end' : 'next',
hasNextPage: !req.variables.endCursor,
},
},
},
}),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const members = await client.getGroupMembers('group1');
expect(members).toEqual([1, 2]);
});
});
});
describe('paginated', () => {
plugins/catalog-backend-module-gitlab/src/lib/client.ts
+50 -34
View File
@@ -20,7 +20,7 @@ import {
GitLabIntegrationConfig,
} from '@backstage/integration';
import { Logger } from 'winston';
import { GitLabGroup, GitLabMembership, GitLabUser } from './types';
import { GitLabGroup, GitLabGroupMembersResponse, GitLabUser } from './types';
export type CommonListOptions = {
[key: string]: string | number | boolean | undefined;
@@ -78,16 +78,11 @@ export class GitLabClient {
async listUsers(
options?: UserListOptions,
): Promise<PagedResponse<GitLabUser>> {
let requestOptions = options;
if (!requestOptions) {
requestOptions = {};
}
requestOptions.without_project_bots = true;
requestOptions.exclude_internal = true;
return this.pagedRequest(`/users?`, requestOptions);
return this.pagedRequest(`/users?`, {
...options,
without_project_bots: true,
exclude_internal: true,
});
}
async listGroups(
@@ -96,30 +91,51 @@ export class GitLabClient {
return this.pagedRequest(`/groups`, options);
}
async getUserMemberships(userId: number): Promise<GitLabMembership[]> {
const endpoint: string = `/users/${encodeURIComponent(userId)}/memberships`;
const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
request.searchParams.append('per_page', '100');
const response = await fetch(request.toString(), {
headers: getGitLabRequestOptions(this.config).headers,
method: 'GET',
});
if (!response.ok) {
if (response.status >= 500) {
this.logger.debug(
`Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
response.status
} - ${response.statusText}`,
);
async getGroupMembers(groupPath: string): Promise<number[]> {
const memberIds = [];
let hasNextPage: boolean = false;
let endCursor: string | null = null;
do {
const response: GitLabGroupMembersResponse = await fetch(
`${this.config.baseUrl}/api/graphql`,
{
method: 'POST',
headers: {
...getGitLabRequestOptions(this.config).headers,
['Content-Type']: 'application/json',
},
body: JSON.stringify({
variables: { group: groupPath, endCursor },
query: `query($group: ID!, $endCursor: String) {
group(fullPath: $group) {
groupMembers(first: 100, relations: [DIRECT], after: $endCursor) {
nodes {
user {
id
}
}
pageInfo {
endCursor
hasNextPage
}
}
}
}`,
}),
},
).then(r => r.json());
if (response.errors) {
throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`);
}
return [];
}
return response.json().then(items => {
return items as GitLabMembership[];
});
memberIds.push(
...response.data.group.groupMembers.nodes.map(
(node: { user: { id: string } }) =>
Number(node.user.id.replace(/^gid:\/\/gitlab\/User\//, '')),
),
);
({ hasNextPage, endCursor } = response.data.group.groupMembers.pageInfo);
} while (hasNextPage);
return memberIds;
}
/**
plugins/catalog-backend-module-gitlab/src/lib/types.ts
+13 -5
View File
@@ -50,11 +50,19 @@ export type GitLabGroup = {
parent_id?: number;
};
export type GitLabMembership = {
source_id: number;
source_name: string;
source_type: string;
access_level: number;
export type GitLabGroupMembersResponse = {
errors: { message: string }[];
data: {
group: {
groupMembers: {
nodes: { user: { id: string } }[];
pageInfo: {
endCursor: string;
hasNextPage: boolean;
};
};
};
};
};
export type GitlabProviderConfig = {
plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts
+21 -15
View File
@@ -23,7 +23,7 @@ import {
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
import { rest } from 'msw';
import { graphql, rest } from 'msw';
import { setupServer } from 'msw/node';
import { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider';
@@ -318,20 +318,26 @@ describe('GitlabOrgDiscoveryEntityProvider', () => {
];
return res(ctx.json(response));
}),
rest.get(
`https://api.gitlab.example/api/v4/users/1/memberships`,
(_req, res, ctx) => {
const response = [
{
source_id: 2,
source_name: 'Group 2',
source_type: 'Namespace',
access_level: 50,
},
];
return res(ctx.json(response));
},
),
graphql
.link('https://test-gitlab/api/graphql')
.operation(async (req, res, ctx) =>
res(
ctx.data({
group: {
groupMembers: {
nodes:
req.variables.group === 'group1/group2'
? [{ user: { id: 'gid://gitlab/User/1' } }]
: [],
pageInfo: {
endCursor: 'end',
hasNextPage: false,
},
},
},
}),
),
),
);
await provider.connect(entityProviderConnection);
plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts
+20 -28
View File
@@ -185,7 +185,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
},
);
const idMappedGroup: { [groupId: number]: GitLabGroup } = {};
const idMappedUser: { [userId: number]: GitLabUser } = {};
const res: Result = {
scanned: 0,
@@ -197,6 +197,21 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
matches: [],
};
for await (const user of users) {
if (!this.config.userPattern.test(user.email ?? user.username ?? '')) {
continue;
}
res.scanned++;
if (user.state !== 'active') {
continue;
}
idMappedUser[user.id] = user;
res.matches.push(user);
}
for await (const group of groups) {
if (!this.config.groupPattern.test(group.full_path ?? '')) {
continue;
@@ -212,35 +227,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
groupRes.scanned++;
groupRes.matches.push(group);
idMappedGroup[group.id] = group;
}
for await (const user of users) {
if (!this.config.userPattern.test(user.email ?? user.username ?? '')) {
continue;
}
res.scanned++;
if (user.state !== 'active') {
continue;
}
const memberships = await client.getUserMemberships(user.id);
const userGroups: GitLabGroup[] = [];
for (const i of memberships) {
if (
i.source_type === 'Namespace' &&
idMappedGroup.hasOwnProperty(i.source_id)
) {
userGroups.push(idMappedGroup[i.source_id]);
for (const id of await client.getGroupMembers(group.full_path)) {
const user = idMappedUser[id];
if (user) {
user.groups = (user.groups ?? []).concat(group);
}
}
user.groups = userGroups;
res.matches.push(user);
}
const groupsWithUsers = groupRes.matches.filter(group => {