paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 42 Packages Projects Releases Wiki Activity

Merge pull request #18889 from angom1/GD-294-users-groups-ingestion

Browse Source 
Fix user ingestion for GitlabOrgDiscoveryEntityProvider for GitLab.com
This commit is contained in:
Fredrik Adelöw
2023-09-12 13:22:54 +02:00
committed by GitHub
parent 35f5c95a35 2c0ac5c2b4
commit 6b1a6e7f84
7 changed files with 935 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/thick-pets-drive.md
+17
View File
@@ -0,0 +1,17 @@
---
'@backstage/plugin-catalog-backend-module-gitlab': minor
---
Fix Gitlab.com user ingestion by scoping GitlabOrgDiscoveryEntityProvider to a group.
**BREAKING** The `group` parameter is now required Gitlab.com Org Data integrations and the backend will fail to start without this option configured.
```diff
catalog:
providers:
gitlab:
yourProviderId:
host: gitlab.com
orgEnabled: true
+ group: org/teams
```
docs/integrations/gitlab/org.md
+42 -1
View File
@@ -65,11 +65,52 @@ catalog:
yourProviderId:
host: gitlab.com
orgEnabled: true
group: org/teams # Optional. Must not end with slash. Accepts only groups under the provided path (which will be stripped)
group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped)
groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything
```
### Groups
When the `group` parameter is provided, the corresponding path prefix will be
stripped out from each matching group when computing the unique entity name.
e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will be
`avengers-gotg`.
For gitlab.com, when `orgEnabled: true`, the `group` parameter is required in
order to limit the ingestion to a group within your organisation. `Group`
entities will only be ingested for the configured group, or its descendant groups,
but not any ancestor groups higher than the configured group path. Only groups
which contain members will be ingested.
### Users
For self hosted, all `User` entities are ingested from the entire instance.
For gitlab.com `User` entities for users who have [direct or inherited membership](https://docs.gitlab.com/ee/user/project/members/index.html#membership-types)
of the top-level group for the configured group path will be ingested.
### Limiting `User` and `Group` entity ingestion in the provider
Optionally, you can limit the entity types ingested by the provider when using
`orgEnabled: true` with the following `rules` configuration to limit it to only
`User` and `Group` entities.
```yaml
catalog:
providers:
gitlab:
yourOrgDataProviderId:
host: gitlab.com
orgEnabled: true
group: org/teams
rules:
- allow: [Group, User]
```
## Troubleshooting
**NOTE**: If any groups that are being ingested are empty groups (i.e. do not
contain any projects) and the user which provisioned the token is shared with a
higher level group via [group sharing](https://docs.gitlab.com/ee/user/group/manage.html#share-a-group-with-another-group)
and you don't see the expected number of `Group` entities in the catalog you may
be hitting this [Gitlab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996).
plugins/catalog-backend-module-gitlab/src/lib/client.test.ts
+391 -15
View File
@@ -398,17 +398,29 @@ describe('GitLabClient', () => {
]);
});
describe('getGroupMembers', () => {
it('gets member IDs', async () => {
describe('get gitlab.com users', () => {
it('gets all users under group', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.operation((_, res, ctx) =>
.query('getGroupMembers', async (_, res, ctx) =>
res(
ctx.data({
group: {
groupMembers: {
nodes: [{ user: { id: 'gid://gitlab/User/1' } }],
nodes: [
{
user: {
id: 'gid://gitlab/User/1',
username: 'user1',
commitEmail: 'user1@example.com',
name: 'user1',
state: 'active',
webUrl: 'user1.com',
avatarUrl: 'user1',
},
},
],
pageInfo: {
endCursor: 'end',
hasNextPage: false,
@@ -424,16 +436,30 @@ describe('GitLabClient', () => {
logger: getVoidLogger(),
});
const members = await client.getGroupMembers('group1');
const saasMembers = (
await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS'])
).items;
const expectedSaasMember = [
{
id: 1,
username: 'user1',
email: 'user1@example.com',
name: 'user1',
state: 'active',
web_url: 'user1.com',
avatar_url: 'user1',
},
];
expect(members).toEqual([1]);
expect(saasMembers.length).toEqual(1);
expect(saasMembers).toEqual(expectedSaasMember);
});
it('gets member IDs with token without full permissions', async () => {
it('gets all users with token without full permissions', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.operation((_, res, ctx) =>
.query('getGroupMembers', async (_, res, ctx) =>
res(
ctx.data({
group: {},
@@ -446,16 +472,18 @@ describe('GitLabClient', () => {
logger: getVoidLogger(),
});
const members = await client.getGroupMembers('group1');
const saasMembers = (
await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS'])
).items;
expect(members).toEqual([]);
expect(saasMembers).toEqual([]);
});
it('rejects when GraphQL returns errors', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.operation((_, res, ctx) =>
.query('getGroupMembers', async (_, res, ctx) =>
res(
ctx.errors([
{ message: 'Unexpected end of document', locations: [] },
@@ -468,7 +496,354 @@ describe('GitLabClient', () => {
logger: getVoidLogger(),
});
await expect(() => client.getGroupMembers('group1')).rejects.toThrow(
await expect(() =>
client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']),
).rejects.toThrow(
'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]',
);
});
it('traverses multi-page results', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.query('getGroupMembers', async (req, res, ctx) =>
res(
ctx.data({
group: {
groupMembers: {
nodes: req.variables.endCursor
? [
{
user: {
id: 'gid://gitlab/User/1',
username: 'user1',
commitEmail: 'user1@example.com',
name: 'user1',
state: 'active',
webUrl: 'user1.com',
avatarUrl: 'user1',
},
},
]
: [
{
user: {
id: 'gid://gitlab/User/2',
username: 'user2',
commitEmail: 'user2@example.com',
name: 'user2',
state: 'active',
webUrl: 'user2.com',
avatarUrl: 'user2',
},
},
],
pageInfo: {
endCursor: req.variables.endCursor ? 'end' : 'next',
hasNextPage: !req.variables.endCursor,
},
},
},
}),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const saasMembers = (
await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS'])
).items;
const expectedSaasMember1 = {
id: 1,
username: 'user1',
email: 'user1@example.com',
name: 'user1',
state: 'active',
web_url: 'user1.com',
avatar_url: 'user1',
};
const expectedSaasMember2 = {
id: 2,
username: 'user2',
email: 'user2@example.com',
name: 'user2',
state: 'active',
web_url: 'user2.com',
avatar_url: 'user2',
};
expect(saasMembers.length).toEqual(2);
expect(saasMembers[0]).toEqual(expectedSaasMember2);
expect(saasMembers[1]).toEqual(expectedSaasMember1);
});
});
describe('listDescendantGroups', () => {
it('gets all groups under root', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.query('listDescendantGroups', async (_, res, ctx) =>
res(
ctx.data({
group: {
descendantGroups: {
nodes: [
{
id: 'gid://gitlab/Group/1',
name: 'group1',
description: 'description1',
fullPath: 'path/group1',
parent: {
id: '123',
},
},
],
pageInfo: {
endCursor: 'end',
hasNextPage: false,
},
},
},
}),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const saasGroups = (await client.listDescendantGroups('group1')).items;
const expectedSaasGroup = [
{
id: 1,
name: 'group1',
description: 'description1',
full_path: 'path/group1',
parent_id: 123,
},
];
expect(saasGroups.length).toEqual(1);
expect(saasGroups).toEqual(expectedSaasGroup);
});
it('gets all descendant groups with token without full permissions', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.query('listDescendantGroups', async (_, res, ctx) =>
res(
ctx.data({
group: {},
}),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const saasGroups = (await client.listDescendantGroups('group1')).items;
expect(saasGroups).toEqual([]);
});
it('rejects when GraphQL returns errors', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.query('listDescendantGroups', async (_, res, ctx) =>
res(
ctx.errors([
{ message: 'Unexpected end of document', locations: [] },
]),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
await expect(() => client.listDescendantGroups('group1')).rejects.toThrow(
'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]',
);
});
it('traverses multi-page results', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.query('listDescendantGroups', async (req, res, ctx) =>
res(
ctx.data({
group: {
descendantGroups: {
nodes: req.variables.endCursor
? [
{
id: 'gid://gitlab/Group/1',
name: 'group1',
description: 'description1',
fullPath: 'path/group1',
parent: {
id: '123',
},
},
]
: [
{
id: 'gid://gitlab/Group/2',
name: 'group2',
description: 'description2',
fullPath: 'path/group2',
parent: {
id: '123',
},
},
],
pageInfo: {
endCursor: req.variables.endCursor ? 'end' : 'next',
hasNextPage: !req.variables.endCursor,
},
},
},
}),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const saasGroups = (await client.listDescendantGroups('root')).items;
const expectedSaasGroup1 = {
id: 1,
name: 'group1',
description: 'description1',
full_path: 'path/group1',
parent_id: 123,
};
const expectedSaasGroup2 = {
id: 2,
name: 'group2',
description: 'description2',
full_path: 'path/group2',
parent_id: 123,
};
expect(saasGroups.length).toEqual(2);
expect(saasGroups[0]).toEqual(expectedSaasGroup2);
expect(saasGroups[1]).toEqual(expectedSaasGroup1);
});
});
describe('getGroupMembers', () => {
it('gets member IDs', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.query('getGroupMembers', async (_, res, ctx) =>
res(
ctx.data({
group: {
groupMembers: {
nodes: [
{
user: {
id: 'gid://gitlab/User/1',
username: 'user1',
commitEmail: 'user1@example.com',
name: 'user1',
state: 'active',
webUrl: 'user1.com',
avatarUrl: 'user1',
},
},
],
pageInfo: {
endCursor: 'end',
hasNextPage: false,
},
},
},
}),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const members = await client.getGroupMembers('group1', ['DIRECT']);
const user = {
id: 1,
username: 'user1',
email: 'user1@example.com',
name: 'user1',
state: 'active',
web_url: 'user1.com',
avatar_url: 'user1',
};
expect(members.items).toEqual([user]);
});
it('gets member IDs with token without full permissions', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.query('getGroupMembers', async (_, res, ctx) =>
res(
ctx.data({
group: {},
}),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
const members = await client.getGroupMembers('group1', ['DIRECT']);
expect(members.items).toEqual([]);
});
it('rejects when GraphQL returns errors', async () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.query('getGroupMembers', async (_, res, ctx) =>
res(
ctx.errors([
{ message: 'Unexpected end of document', locations: [] },
]),
),
),
);
const client = new GitLabClient({
config: MOCK_CONFIG,
logger: getVoidLogger(),
});
await expect(() =>
client.getGroupMembers('group1', ['DIRECT']),
).rejects.toThrow(
'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]',
);
});
@@ -477,7 +852,7 @@ describe('GitLabClient', () => {
server.use(
graphql
.link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
.operation((req, res, ctx) =>
.query('getGroupMembers', async (req, res, ctx) =>
res(
ctx.data({
group: {
@@ -500,9 +875,10 @@ describe('GitLabClient', () => {
logger: getVoidLogger(),
});
const members = await client.getGroupMembers('group1');
const members = await client.getGroupMembers('group1', ['DIRECT']);
expect(members).toEqual([1, 2]);
expect(members.items[0].id).toEqual(1);
expect(members.items[1].id).toEqual(2);
});
});
});
plugins/catalog-backend-module-gitlab/src/lib/client.ts
+128 -22
View File
@@ -20,7 +20,12 @@ import {
GitLabIntegrationConfig,
} from '@backstage/integration';
import { Logger } from 'winston';
import { GitLabGroup, GitLabGroupMembersResponse, GitLabUser } from './types';
import {
GitLabGroup,
GitLabDescendantGroupsResponse,
GitLabGroupMembersResponse,
GitLabUser,
} from './types';
export type CommonListOptions = {
[key: string]: string | number | boolean | undefined;
@@ -92,12 +97,15 @@ export class GitLabClient {
return this.pagedRequest(`/groups`, options);
}
async getGroupMembers(groupPath: string): Promise<number[]> {
const memberIds = [];
async listDescendantGroups(
groupPath: string,
): Promise<PagedResponse<GitLabGroup>> {
const items: GitLabGroup[] = [];
let hasNextPage: boolean = false;
let endCursor: string | null = null;
do {
const response: GitLabGroupMembersResponse = await fetch(
const response: GitLabDescendantGroupsResponse = await fetch(
`${this.config.baseUrl}/api/graphql`,
{
method: 'POST',
@@ -107,21 +115,111 @@ export class GitLabClient {
},
body: JSON.stringify({
variables: { group: groupPath, endCursor },
query: `query($group: ID!, $endCursor: String) {
group(fullPath: $group) {
groupMembers(first: 100, relations: [DIRECT], after: $endCursor) {
nodes {
user {
query: /* GraphQL */ `
query listDescendantGroups($group: ID!, $endCursor: String) {
group(fullPath: $group) {
descendantGroups(first: 100, after: $endCursor) {
nodes {
id
name
description
fullPath
parent {
id
}
}
pageInfo {
endCursor
hasNextPage
}
}
pageInfo {
endCursor
hasNextPage
}
}
}
}`,
`,
}),
},
).then(r => r.json());
if (response.errors) {
throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`);
}
if (!response.data.group?.descendantGroups?.nodes) {
this.logger.warn(
`Couldn't get groups under ${groupPath}. The provided token might not have sufficient permissions`,
);
continue;
}
for (const groupItem of response.data.group.descendantGroups.nodes.filter(
group => group?.id,
)) {
const formattedGroupResponse = {
id: Number(groupItem.id.replace(/^gid:\/\/gitlab\/Group\//, '')),
name: groupItem.name,
description: groupItem.description,
full_path: groupItem.fullPath,
parent_id: Number(
groupItem.parent.id.replace(/^gid:\/\/gitlab\/Group\//, ''),
),
};
items.push(formattedGroupResponse);
}
({ hasNextPage, endCursor } =
response.data.group.descendantGroups.pageInfo);
} while (hasNextPage);
return { items };
}
async getGroupMembers(
groupPath: string,
relations: string[],
): Promise<PagedResponse<GitLabUser>> {
const items: GitLabUser[] = [];
let hasNextPage: boolean = false;
let endCursor: string | null = null;
do {
const response: GitLabGroupMembersResponse = await fetch(
`${this.config.baseUrl}/api/graphql`,
{
method: 'POST',
headers: {
...getGitLabRequestOptions(this.config).headers,
'Content-Type': 'application/json',
},
body: JSON.stringify({
variables: { group: groupPath, relations: relations, endCursor },
query: /* GraphQL */ `
query getGroupMembers(
$group: ID!
$relations: [GroupMemberRelation!]
$endCursor: String
) {
group(fullPath: $group) {
groupMembers(
first: 100
relations: $relations
after: $endCursor
) {
nodes {
user {
id
username
commitEmail
name
state
webUrl
avatarUrl
}
}
pageInfo {
endCursor
hasNextPage
}
}
}
}
`,
}),
},
).then(r => r.json());
@@ -136,16 +234,24 @@ export class GitLabClient {
continue;
}
memberIds.push(
...response.data.group.groupMembers.nodes
.filter(n => n.user)
.map(node =>
Number(node.user.id.replace(/^gid:\/\/gitlab\/User\//, '')),
),
);
for (const userItem of response.data.group.groupMembers.nodes.filter(
user => user.user?.id,
)) {
const formattedUserResponse = {
id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')),
username: userItem.user.username,
email: userItem.user.commitEmail,
name: userItem.user.name,
state: userItem.user.state,
web_url: userItem.user.webUrl,
avatar_url: userItem.user.avatarUrl,
};
items.push(formattedUserResponse);
}
({ hasNextPage, endCursor } = response.data.group.groupMembers.pageInfo);
} while (hasNextPage);
return memberIds;
return { items };
}
/**
plugins/catalog-backend-module-gitlab/src/lib/types.ts
+36 -1
View File
@@ -60,7 +60,17 @@ export type GitLabGroupMembersResponse = {
data: {
group: {
groupMembers: {
nodes: { user: { id: string } }[];
nodes: {
user: {
id: string;
username: string;
commitEmail: string;
name: string;
state: string;
webUrl: string;
avatarUrl: string;
};
}[];
pageInfo: {
endCursor: string;
hasNextPage: boolean;
@@ -70,6 +80,31 @@ export type GitLabGroupMembersResponse = {
};
};
export type GitLabDescendantGroupsResponse = {
errors: { message: string }[];
data: {
group: {
descendantGroups: {
nodes: [
{
id: string;
name: string;
description: string;
fullPath: string;
parent: {
id: string;
};
},
];
pageInfo: {
endCursor: string;
hasNextPage: false;
};
};
};
};
};
export type GitlabProviderConfig = {
host: string;
group: string;
plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts
+289
View File
@@ -475,6 +475,259 @@ describe('GitlabOrgDiscoveryEntityProvider', () => {
});
});
it('apply full update on scheduled execution for gitlab.com', async () => {
const config = new ConfigReader({
integrations: {
gitlab: [
{
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
token: '1234',
},
],
},
catalog: {
providers: {
gitlab: {
'test-id': {
host: 'gitlab.com',
group: 'group1',
orgEnabled: true,
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
expect(provider.getProviderName()).toEqual(
'GitlabOrgDiscoveryEntityProvider:test-id',
);
server.use(
graphql
.link('https://gitlab.com/api/graphql')
.query('listDescendantGroups', async (_, res, ctx) =>
res(
ctx.data({
group: {
descendantGroups: {
nodes: [
{
id: 'gid://gitlab/Group/456',
name: 'group2',
description: 'Group2',
fullPath: 'group1/group2',
parent: {
id: 'gid://gitlab/Group/123',
},
},
{
id: 'gid://gitlab/Group/789',
name: 'group3',
description: 'Group3',
fullPath: 'group1/group3',
parent: {
id: 'gid://gitlab/Group/123',
},
},
],
pageInfo: {
endCursor: 'end',
hasNextPage: false,
},
},
},
}),
),
),
graphql
.link('https://gitlab.com/api/graphql')
.query('getGroupMembers', async (_, res, ctx) =>
res(
ctx.data({
group: {
groupMembers: {
nodes: [
{
user: {
id: 'gid://gitlab/User/12',
username: 'testuser1',
commitEmail: 'testuser1@example.com',
state: 'active',
name: 'Test User 1',
webUrl: 'https://gitlab.com/testuser1',
avatarUrl: 'https://secure.gravatar.com/',
},
},
{
user: {
id: 'gid://gitlab/User/34',
username: 'testuser2',
commitEmail: 'testuser2@example.com',
state: 'active',
name: 'Test User 2',
webUrl: 'https://gitlab.com/testuser2',
avatarUrl: 'https://secure.gravatar.com/',
},
},
],
pageInfo: {
endCursor: 'end',
hasNextPage: false,
},
},
},
}),
),
),
graphql
.link('https://gitlab.com/api/graphql')
.query('getGroupMembers', async (req, res, ctx) =>
res(
ctx.data({
group: {
groupMembers: {
nodes:
req.variables.group === 'group1/group2'
? [{ user: { id: 'gid://gitlab/User/12' } }]
: [{ user: { id: 'gid://gitlab/User/34' } }],
pageInfo: {
endCursor: 'end',
hasNextPage: false,
},
},
},
}),
),
),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual(
'GitlabOrgDiscoveryEntityProvider:test-id:refresh',
);
await (taskDef.fn as () => Promise<void>)();
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://gitlab.com/testuser1',
'backstage.io/managed-by-origin-location':
'url:https://gitlab.com/testuser1',
'gitlab.com/user-login': 'https://gitlab.com/testuser1',
},
name: 'testuser1',
},
spec: {
memberOf: ['group2', 'group3'],
profile: {
displayName: 'Test User 1',
email: 'testuser1@example.com',
picture: 'https://secure.gravatar.com/',
},
},
},
locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://gitlab.com/testuser2',
'backstage.io/managed-by-origin-location':
'url:https://gitlab.com/testuser2',
'gitlab.com/user-login': 'https://gitlab.com/testuser2',
},
name: 'testuser2',
},
spec: {
memberOf: ['group2', 'group3'],
profile: {
displayName: 'Test User 2',
email: 'testuser2@example.com',
picture: 'https://secure.gravatar.com/',
},
},
},
locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://gitlab.com/group1/group2',
'backstage.io/managed-by-origin-location':
'url:https://gitlab.com/group1/group2',
'gitlab.com/team-path': 'group1/group2',
},
description: 'Group2',
name: 'group2',
},
spec: {
children: [],
profile: {
displayName: 'group2',
},
type: 'team',
},
},
locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://gitlab.com/group1/group3',
'backstage.io/managed-by-origin-location':
'url:https://gitlab.com/group1/group3',
'gitlab.com/team-path': 'group1/group3',
},
description: 'Group3',
name: 'group3',
},
spec: {
children: [],
profile: {
displayName: 'group3',
},
type: 'team',
},
},
locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id',
},
];
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
});
it('fail without schedule and scheduler', () => {
const config = new ConfigReader({
integrations: {
@@ -543,6 +796,42 @@ describe('GitlabOrgDiscoveryEntityProvider', () => {
);
});
it('fail with scheduler but no group config when host is gitlab.com', () => {
const scheduler = {
createScheduledTaskRunner: (_: any) => jest.fn(),
} as unknown as PluginTaskScheduler;
const config = new ConfigReader({
integrations: {
gitlab: [
{
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
token: '1234',
},
],
},
catalog: {
providers: {
gitlab: {
'test-id': {
host: 'gitlab.com',
orgEnabled: true,
},
},
},
},
});
expect(() =>
GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
).toThrow(
`Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`,
);
});
it('single simple provider config with schedule in config', async () => {
const schedule = new PersistingTaskRunner();
const scheduler = {
plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts
+32 -12
View File
@@ -89,6 +89,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
);
}
if (!providerConfig.group && providerConfig.host === 'gitlab.com') {
throw new Error(
`Missing 'group' value for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`,
);
}
if (!options.schedule && !providerConfig.schedule) {
throw new Error(
`No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`,
@@ -168,19 +174,29 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
logger: logger,
});
const users = paginated<GitLabUser>(options => client.listUsers(options), {
page: 1,
per_page: 100,
active: true,
});
let groups;
let users;
const groups = paginated<GitLabGroup>(
options => client.listGroups(options),
{
if (client.isSelfManaged()) {
groups = paginated<GitLabGroup>(options => client.listGroups(options), {
page: 1,
per_page: 100,
},
);
});
users = paginated<GitLabUser>(options => client.listUsers(options), {
page: 1,
per_page: 100,
active: true,
});
} else {
groups = (await client.listDescendantGroups(this.config.group)).items;
users = (
await client.getGroupMembers(this.config.group.split('/')[0], [
'DIRECT',
'DESCENDANTS',
])
).items;
}
const idMappedUser: { [userId: number]: GitLabUser } = {};
@@ -224,8 +240,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
groupRes.scanned++;
groupRes.matches.push(group);
for (const id of await client.getGroupMembers(group.full_path)) {
const user = idMappedUser[id];
const groupUsers = await client.getGroupMembers(group.full_path, [
'DIRECT',
]);
for (const groupUser of groupUsers.items) {
const user = idMappedUser[groupUser.id];
if (user) {
user.groups = (user.groups ?? []).concat(group);
}