Merge pull request #15888 from dpfaffenbauer/gitlab-org-catalog-provider
Implement gitlab Organization catalog provider for Users and Groups
This commit is contained in:
Ben Lambert
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-gitlab': patch
|
||||
---
|
||||
|
||||
Implement Group and User Catalog Provider
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
id: org
|
||||
title: GitLab Org
|
||||
sidebar_label: Org Data
|
||||
description: Importing users and groups from a GitLab organization into Backstage
|
||||
---
|
||||
|
||||
The Backstage catalog can be set up to ingest organizational data - users and
|
||||
teams - directly from an organization in GitLab. The result
|
||||
is a hierarchy of
|
||||
[`User`](../../features/software-catalog/descriptor-format.md#kind-user) and
|
||||
[`Group`](../../features/software-catalog/descriptor-format.md#kind-group) kind
|
||||
entities that mirror your org setup.
|
||||
|
||||
```yaml
|
||||
integrations:
|
||||
gitlab:
|
||||
- host: gitlab.com
|
||||
token: ${GITLAB_TOKEN}
|
||||
```
|
||||
|
||||
This will query all users and groups from your gitlab installation. Depending on the size
|
||||
of the Gitlab Instance, this can take some time our resources.
|
||||
|
||||
The token that is used for the Organization Integration, has to be an Admin PAT.
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
providers:
|
||||
gitlab:
|
||||
yourProviderId:
|
||||
host: gitlab.com
|
||||
orgEnabled: true
|
||||
```
|
||||
@@ -55,4 +55,21 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
|
||||
emit: CatalogProcessorEmit,
|
||||
): Promise<boolean>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
|
||||
// (undocumented)
|
||||
connect(connection: EntityProviderConnection): Promise<void>;
|
||||
// (undocumented)
|
||||
static fromConfig(
|
||||
config: Config,
|
||||
options: {
|
||||
logger: Logger;
|
||||
schedule?: TaskRunner;
|
||||
scheduler?: PluginTaskScheduler;
|
||||
},
|
||||
): GitlabOrgDiscoveryEntityProvider[];
|
||||
// (undocumented)
|
||||
getProviderName(): string;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -48,6 +48,18 @@ export interface Config {
|
||||
* (Optional) TaskScheduleDefinition for the refresh.
|
||||
*/
|
||||
schedule?: TaskScheduleDefinitionConfig;
|
||||
/**
|
||||
* (Optional) RegExp for the Project Name Pattern
|
||||
*/
|
||||
projectPattern?: RegExp;
|
||||
/**
|
||||
* (Optional) RegExp for the User Name Pattern
|
||||
*/
|
||||
userPattern?: RegExp;
|
||||
/**
|
||||
* (Optional) RegExp for the Group Name Pattern
|
||||
*/
|
||||
groupPattern?: RegExp;
|
||||
}
|
||||
>;
|
||||
};
|
||||
|
||||
@@ -21,5 +21,8 @@
|
||||
*/
|
||||
|
||||
export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor';
|
||||
export { GitlabDiscoveryEntityProvider } from './providers';
|
||||
export {
|
||||
GitlabDiscoveryEntityProvider,
|
||||
GitlabOrgDiscoveryEntityProvider,
|
||||
} from './providers';
|
||||
export { gitlabDiscoveryEntityProviderCatalogModule } from './service/GitlabDiscoveryEntityProviderCatalogModule';
|
||||
|
||||
@@ -20,14 +20,24 @@ import {
|
||||
GitLabIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import { Logger } from 'winston';
|
||||
import { GitLabGroup, GitLabMembership, GitLabUser } from './types';
|
||||
|
||||
export type ListOptions = {
|
||||
export type CommonListOptions = {
|
||||
[key: string]: string | number | boolean | undefined;
|
||||
group?: string;
|
||||
per_page?: number | undefined;
|
||||
page?: number | undefined;
|
||||
active?: boolean;
|
||||
};
|
||||
|
||||
interface ListProjectOptions extends CommonListOptions {
|
||||
group?: string;
|
||||
}
|
||||
|
||||
interface UserListOptions extends CommonListOptions {
|
||||
without_project_bots?: boolean | undefined;
|
||||
exclude_internal?: boolean | undefined;
|
||||
}
|
||||
|
||||
export type PagedResponse<T> = {
|
||||
items: T[];
|
||||
nextPage?: number;
|
||||
@@ -49,7 +59,9 @@ export class GitLabClient {
|
||||
return this.config.host !== 'gitlab.com';
|
||||
}
|
||||
|
||||
async listProjects(options?: ListOptions): Promise<PagedResponse<any>> {
|
||||
async listProjects(
|
||||
options?: ListProjectOptions,
|
||||
): Promise<PagedResponse<any>> {
|
||||
if (options?.group) {
|
||||
return this.pagedRequest(
|
||||
`/groups/${encodeURIComponent(options?.group)}/projects`,
|
||||
@@ -63,6 +75,53 @@ export class GitLabClient {
|
||||
return this.pagedRequest(`/projects`, options);
|
||||
}
|
||||
|
||||
async listUsers(
|
||||
options?: UserListOptions,
|
||||
): Promise<PagedResponse<GitLabUser>> {
|
||||
let requestOptions = options;
|
||||
|
||||
if (!requestOptions) {
|
||||
requestOptions = {};
|
||||
}
|
||||
|
||||
requestOptions.without_project_bots = true;
|
||||
requestOptions.exclude_internal = true;
|
||||
|
||||
return this.pagedRequest(`/users?`, requestOptions);
|
||||
}
|
||||
|
||||
async listGroups(
|
||||
options?: CommonListOptions,
|
||||
): Promise<PagedResponse<GitLabGroup>> {
|
||||
return this.pagedRequest(`/groups`, options);
|
||||
}
|
||||
|
||||
async getUserMemberships(userId: number): Promise<GitLabMembership[]> {
|
||||
const endpoint: string = `/users/${encodeURIComponent(userId)}/memberships`;
|
||||
const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
|
||||
request.searchParams.append('per_page', '100');
|
||||
|
||||
const response = await fetch(request.toString(), {
|
||||
headers: getGitLabRequestOptions(this.config).headers,
|
||||
method: 'GET',
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status >= 500) {
|
||||
this.logger.debug(
|
||||
`Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
|
||||
response.status
|
||||
} - ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
return response.json().then(items => {
|
||||
return items as GitLabMembership[];
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* General existence check.
|
||||
*
|
||||
@@ -114,7 +173,7 @@ export class GitLabClient {
|
||||
*/
|
||||
async pagedRequest<T = any>(
|
||||
endpoint: string,
|
||||
options?: ListOptions,
|
||||
options?: CommonListOptions,
|
||||
): Promise<PagedResponse<T>> {
|
||||
const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
|
||||
for (const key in options) {
|
||||
@@ -159,8 +218,8 @@ export class GitLabClient {
|
||||
* @param options - Initial ListOptions for the request function.
|
||||
*/
|
||||
export async function* paginated<T = any>(
|
||||
request: (options: ListOptions) => Promise<PagedResponse<T>>,
|
||||
options: ListOptions,
|
||||
request: (options: CommonListOptions) => Promise<PagedResponse<T>>,
|
||||
options: CommonListOptions,
|
||||
) {
|
||||
let res;
|
||||
do {
|
||||
|
||||
@@ -31,6 +31,32 @@ export type GitLabProject = {
|
||||
path_with_namespace?: string;
|
||||
};
|
||||
|
||||
export type GitLabUser = {
|
||||
id: number;
|
||||
username: string;
|
||||
email: string;
|
||||
name: string;
|
||||
state: string;
|
||||
web_url: string;
|
||||
avatar_url: string;
|
||||
groups?: GitLabGroup[];
|
||||
};
|
||||
|
||||
export type GitLabGroup = {
|
||||
id: number;
|
||||
name: string;
|
||||
full_path: string;
|
||||
description?: string;
|
||||
parent_id?: number;
|
||||
};
|
||||
|
||||
export type GitLabMembership = {
|
||||
source_id: number;
|
||||
source_name: string;
|
||||
source_type: string;
|
||||
access_level: number;
|
||||
};
|
||||
|
||||
export type GitlabProviderConfig = {
|
||||
host: string;
|
||||
group: string;
|
||||
@@ -38,5 +64,8 @@ export type GitlabProviderConfig = {
|
||||
branch: string;
|
||||
catalogFile: string;
|
||||
projectPattern: RegExp;
|
||||
userPattern: RegExp;
|
||||
groupPattern: RegExp;
|
||||
orgEnabled?: boolean;
|
||||
schedule?: TaskScheduleDefinition;
|
||||
};
|
||||
|
||||
+514
@@ -0,0 +1,514 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import {
|
||||
PluginTaskScheduler,
|
||||
TaskInvocationDefinition,
|
||||
TaskRunner,
|
||||
} from '@backstage/backend-tasks';
|
||||
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider';
|
||||
|
||||
class PersistingTaskRunner implements TaskRunner {
|
||||
private tasks: TaskInvocationDefinition[] = [];
|
||||
|
||||
getTasks() {
|
||||
return this.tasks;
|
||||
}
|
||||
|
||||
run(task: TaskInvocationDefinition): Promise<void> {
|
||||
this.tasks.push(task);
|
||||
return Promise.resolve(undefined);
|
||||
}
|
||||
}
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
const server = setupServer();
|
||||
|
||||
describe('GitlabOrgDiscoveryEntityProvider', () => {
|
||||
setupRequestMockHandlers(server);
|
||||
afterEach(() => jest.resetAllMocks());
|
||||
|
||||
it('no provider config', () => {
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const config = new ConfigReader({});
|
||||
const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule,
|
||||
});
|
||||
|
||||
expect(providers).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('single simple discovery config with org disabled', () => {
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const config = new ConfigReader({
|
||||
integrations: {
|
||||
gitlab: [
|
||||
{
|
||||
host: 'test-gitlab',
|
||||
apiBaseUrl: 'https://api.gitlab.example/api/v4',
|
||||
token: '1234',
|
||||
},
|
||||
],
|
||||
},
|
||||
catalog: {
|
||||
providers: {
|
||||
gitlab: {
|
||||
'test-id': {
|
||||
host: 'test-gitlab',
|
||||
group: 'test-group',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule,
|
||||
});
|
||||
|
||||
expect(providers).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('single simple discovery config with org enabled', () => {
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const config = new ConfigReader({
|
||||
integrations: {
|
||||
gitlab: [
|
||||
{
|
||||
host: 'test-gitlab',
|
||||
apiBaseUrl: 'https://api.gitlab.example/api/v4',
|
||||
token: '1234',
|
||||
},
|
||||
],
|
||||
},
|
||||
catalog: {
|
||||
providers: {
|
||||
gitlab: {
|
||||
'test-id': {
|
||||
host: 'test-gitlab',
|
||||
group: 'test-group',
|
||||
orgEnabled: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule,
|
||||
});
|
||||
|
||||
expect(providers).toHaveLength(1);
|
||||
expect(providers[0].getProviderName()).toEqual(
|
||||
'GitlabOrgDiscoveryEntityProvider:test-id',
|
||||
);
|
||||
});
|
||||
|
||||
it('multiple discovery configs', () => {
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const config = new ConfigReader({
|
||||
integrations: {
|
||||
gitlab: [
|
||||
{
|
||||
host: 'test-gitlab',
|
||||
apiBaseUrl: 'https://api.gitlab.example/api/v4',
|
||||
token: '1234',
|
||||
},
|
||||
],
|
||||
},
|
||||
catalog: {
|
||||
providers: {
|
||||
gitlab: {
|
||||
'test-id': {
|
||||
host: 'test-gitlab',
|
||||
group: 'test-group',
|
||||
orgEnabled: true,
|
||||
},
|
||||
'second-test': {
|
||||
host: 'test-gitlab',
|
||||
group: 'second-group',
|
||||
orgEnabled: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule,
|
||||
});
|
||||
|
||||
expect(providers).toHaveLength(2);
|
||||
expect(providers[0].getProviderName()).toEqual(
|
||||
'GitlabOrgDiscoveryEntityProvider:test-id',
|
||||
);
|
||||
expect(providers[1].getProviderName()).toEqual(
|
||||
'GitlabOrgDiscoveryEntityProvider:second-test',
|
||||
);
|
||||
});
|
||||
|
||||
it('apply full update on scheduled execution', async () => {
|
||||
const config = new ConfigReader({
|
||||
integrations: {
|
||||
gitlab: [
|
||||
{
|
||||
host: 'test-gitlab',
|
||||
apiBaseUrl: 'https://api.gitlab.example/api/v4',
|
||||
token: '1234',
|
||||
},
|
||||
],
|
||||
},
|
||||
catalog: {
|
||||
providers: {
|
||||
gitlab: {
|
||||
'test-id': {
|
||||
host: 'test-gitlab',
|
||||
group: 'test-group',
|
||||
orgEnabled: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule,
|
||||
})[0];
|
||||
expect(provider.getProviderName()).toEqual(
|
||||
'GitlabOrgDiscoveryEntityProvider:test-id',
|
||||
);
|
||||
|
||||
server.use(
|
||||
rest.get(
|
||||
`https://api.gitlab.example/api/v4/groups/test-group/projects`,
|
||||
(_req, res, ctx) => {
|
||||
const response = [
|
||||
{
|
||||
id: 123,
|
||||
default_branch: 'master',
|
||||
archived: false,
|
||||
last_activity_at: new Date().toString(),
|
||||
web_url: 'https://api.gitlab.example/test-group/test-repo',
|
||||
path_with_namespace: 'test-group/test-repo',
|
||||
},
|
||||
];
|
||||
return res(ctx.json(response));
|
||||
},
|
||||
),
|
||||
rest.get(`https://api.gitlab.example/api/v4/users`, (_req, res, ctx) => {
|
||||
const response = [
|
||||
{
|
||||
id: 1,
|
||||
username: 'test1',
|
||||
name: 'Test Testit',
|
||||
state: 'active',
|
||||
avatar_url: 'https://secure.gravatar.com/',
|
||||
web_url: 'https://gitlab.example/test1',
|
||||
created_at: '2023-01-19T07:27:03.333Z',
|
||||
bio: '',
|
||||
location: null,
|
||||
public_email: null,
|
||||
skype: '',
|
||||
linkedin: '',
|
||||
twitter: '',
|
||||
website_url: '',
|
||||
organization: null,
|
||||
job_title: '',
|
||||
pronouns: null,
|
||||
bot: false,
|
||||
work_information: null,
|
||||
followers: 0,
|
||||
following: 0,
|
||||
is_followed: false,
|
||||
local_time: null,
|
||||
last_sign_in_at: '2023-01-19T07:27:49.601Z',
|
||||
confirmed_at: '2023-01-19T07:27:02.905Z',
|
||||
last_activity_on: '2023-01-19',
|
||||
email: 'test@example.com',
|
||||
theme_id: 1,
|
||||
color_scheme_id: 1,
|
||||
projects_limit: 100000,
|
||||
current_sign_in_at: '2023-01-19T09:09:10.676Z',
|
||||
identities: [],
|
||||
can_create_group: true,
|
||||
can_create_project: true,
|
||||
two_factor_enabled: false,
|
||||
external: false,
|
||||
private_profile: false,
|
||||
commit_email: 'test@example.com',
|
||||
is_admin: false,
|
||||
note: '',
|
||||
},
|
||||
];
|
||||
return res(ctx.json(response));
|
||||
}),
|
||||
rest.get(`https://api.gitlab.example/api/v4/groups`, (_req, res, ctx) => {
|
||||
const response = [
|
||||
{
|
||||
id: 1,
|
||||
web_url: 'https://gitlab.example/groups/group1',
|
||||
name: 'group1',
|
||||
path: 'group1',
|
||||
description: '',
|
||||
visibility: 'internal',
|
||||
share_with_group_lock: false,
|
||||
require_two_factor_authentication: false,
|
||||
two_factor_grace_period: 48,
|
||||
project_creation_level: 'developer',
|
||||
auto_devops_enabled: null,
|
||||
subgroup_creation_level: 'owner',
|
||||
emails_disabled: null,
|
||||
mentions_disabled: null,
|
||||
lfs_enabled: true,
|
||||
default_branch_protection: 2,
|
||||
avatar_url: null,
|
||||
request_access_enabled: false,
|
||||
full_name: '8020',
|
||||
full_path: '8020',
|
||||
created_at: '2017-06-19T06:42:34.160Z',
|
||||
parent_id: null,
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
web_url: 'https://gitlab.example/groups/group1/group2',
|
||||
name: 'group2',
|
||||
path: 'group1/group2',
|
||||
description: 'Group2',
|
||||
visibility: 'internal',
|
||||
share_with_group_lock: false,
|
||||
require_two_factor_authentication: false,
|
||||
two_factor_grace_period: 48,
|
||||
project_creation_level: 'developer',
|
||||
auto_devops_enabled: null,
|
||||
subgroup_creation_level: 'owner',
|
||||
emails_disabled: null,
|
||||
mentions_disabled: null,
|
||||
lfs_enabled: true,
|
||||
request_access_enabled: false,
|
||||
full_name: 'group2',
|
||||
full_path: 'group1/group2',
|
||||
created_at: '2017-12-07T13:20:40.675Z',
|
||||
parent_id: null,
|
||||
},
|
||||
];
|
||||
return res(ctx.json(response));
|
||||
}),
|
||||
rest.get(
|
||||
`https://api.gitlab.example/api/v4/users/1/memberships`,
|
||||
(_req, res, ctx) => {
|
||||
const response = [
|
||||
{
|
||||
source_id: 2,
|
||||
source_name: 'Group 2',
|
||||
source_type: 'Namespace',
|
||||
access_level: 50,
|
||||
},
|
||||
];
|
||||
return res(ctx.json(response));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const taskDef = schedule.getTasks()[0];
|
||||
expect(taskDef.id).toEqual(
|
||||
'GitlabOrgDiscoveryEntityProvider:test-id:refresh',
|
||||
);
|
||||
await (taskDef.fn as () => Promise<void>)();
|
||||
|
||||
const expectedEntities = [
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location': 'url:test-gitlab/test1',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:test-gitlab/test1',
|
||||
'test-gitlab/user-login': 'https://gitlab.example/test1',
|
||||
},
|
||||
name: 'test1',
|
||||
},
|
||||
spec: {
|
||||
memberOf: ['group1-group2'],
|
||||
profile: {
|
||||
displayName: 'Test Testit',
|
||||
email: 'test@example.com',
|
||||
picture: 'https://secure.gravatar.com/',
|
||||
},
|
||||
},
|
||||
},
|
||||
locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id',
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Group',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:test-gitlab/teams/group1-group2',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:test-gitlab/teams/group1-group2',
|
||||
'test-gitlab/team-path': 'group1/group2',
|
||||
},
|
||||
description: 'Group2',
|
||||
name: 'group1-group2',
|
||||
},
|
||||
spec: {
|
||||
children: [],
|
||||
profile: {
|
||||
displayName: 'group2',
|
||||
},
|
||||
type: 'team',
|
||||
},
|
||||
},
|
||||
locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id',
|
||||
},
|
||||
];
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'full',
|
||||
entities: expectedEntities,
|
||||
});
|
||||
});
|
||||
|
||||
it('fail without schedule and scheduler', () => {
|
||||
const config = new ConfigReader({
|
||||
integrations: {
|
||||
gitlab: [
|
||||
{
|
||||
host: 'test-gitlab',
|
||||
apiBaseUrl: 'https://api.gitlab.example/api/v4',
|
||||
token: '1234',
|
||||
},
|
||||
],
|
||||
},
|
||||
catalog: {
|
||||
providers: {
|
||||
gitlab: {
|
||||
'test-id': {
|
||||
host: 'test-gitlab',
|
||||
group: 'test-group',
|
||||
orgEnabled: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
}),
|
||||
).toThrow('Either schedule or scheduler must be provided');
|
||||
});
|
||||
|
||||
it('fail with scheduler but no schedule config', () => {
|
||||
const scheduler = {
|
||||
createScheduledTaskRunner: (_: any) => jest.fn(),
|
||||
} as unknown as PluginTaskScheduler;
|
||||
const config = new ConfigReader({
|
||||
integrations: {
|
||||
gitlab: [
|
||||
{
|
||||
host: 'test-gitlab',
|
||||
apiBaseUrl: 'https://api.gitlab.example/api/v4',
|
||||
token: '1234',
|
||||
},
|
||||
],
|
||||
},
|
||||
catalog: {
|
||||
providers: {
|
||||
gitlab: {
|
||||
'test-id': {
|
||||
host: 'test-gitlab',
|
||||
group: 'test-group',
|
||||
orgEnabled: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
scheduler,
|
||||
}),
|
||||
).toThrow(
|
||||
'No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:test-id',
|
||||
);
|
||||
});
|
||||
|
||||
it('single simple provider config with schedule in config', async () => {
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const scheduler = {
|
||||
createScheduledTaskRunner: (_: any) => schedule,
|
||||
} as unknown as PluginTaskScheduler;
|
||||
const config = new ConfigReader({
|
||||
integrations: {
|
||||
gitlab: [
|
||||
{
|
||||
host: 'test-gitlab',
|
||||
apiBaseUrl: 'https://api.gitlab.example/api/v4',
|
||||
token: '1234',
|
||||
},
|
||||
],
|
||||
},
|
||||
catalog: {
|
||||
providers: {
|
||||
gitlab: {
|
||||
'test-id': {
|
||||
host: 'test-gitlab',
|
||||
group: 'test-group',
|
||||
orgEnabled: true,
|
||||
schedule: {
|
||||
frequency: 'PT30M',
|
||||
timeout: 'PT3M',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
scheduler,
|
||||
});
|
||||
|
||||
expect(providers).toHaveLength(1);
|
||||
expect(providers[0].getProviderName()).toEqual(
|
||||
'GitlabOrgDiscoveryEntityProvider:test-id',
|
||||
);
|
||||
});
|
||||
});
|
||||
+375
@@ -0,0 +1,375 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
|
||||
import { Config } from '@backstage/config';
|
||||
import { GitLabIntegration, ScmIntegrations } from '@backstage/integration';
|
||||
import {
|
||||
EntityProvider,
|
||||
EntityProviderConnection,
|
||||
} from '@backstage/plugin-catalog-backend';
|
||||
import * as uuid from 'uuid';
|
||||
import { Logger } from 'winston';
|
||||
import {
|
||||
GitLabClient,
|
||||
GitlabProviderConfig,
|
||||
paginated,
|
||||
readGitlabConfigs,
|
||||
} from '../lib';
|
||||
import { GitLabGroup, GitLabUser } from '../lib/types';
|
||||
import {
|
||||
ANNOTATION_LOCATION,
|
||||
ANNOTATION_ORIGIN_LOCATION,
|
||||
Entity,
|
||||
UserEntity,
|
||||
GroupEntity,
|
||||
} from '@backstage/catalog-model';
|
||||
import { merge } from 'lodash';
|
||||
|
||||
type Result = {
|
||||
scanned: number;
|
||||
matches: GitLabUser[];
|
||||
};
|
||||
|
||||
type GroupResult = {
|
||||
scanned: number;
|
||||
matches: GitLabGroup[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Discovers users and groups from a Gitlab instance.
|
||||
* @public
|
||||
*/
|
||||
export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
|
||||
private readonly config: GitlabProviderConfig;
|
||||
private readonly integration: GitLabIntegration;
|
||||
private readonly logger: Logger;
|
||||
private readonly scheduleFn: () => Promise<void>;
|
||||
private connection?: EntityProviderConnection;
|
||||
|
||||
static fromConfig(
|
||||
config: Config,
|
||||
options: {
|
||||
logger: Logger;
|
||||
schedule?: TaskRunner;
|
||||
scheduler?: PluginTaskScheduler;
|
||||
},
|
||||
): GitlabOrgDiscoveryEntityProvider[] {
|
||||
if (!options.schedule && !options.scheduler) {
|
||||
throw new Error('Either schedule or scheduler must be provided.');
|
||||
}
|
||||
|
||||
const providerConfigs = readGitlabConfigs(config);
|
||||
const integrations = ScmIntegrations.fromConfig(config).gitlab;
|
||||
const providers: GitlabOrgDiscoveryEntityProvider[] = [];
|
||||
|
||||
providerConfigs.forEach(providerConfig => {
|
||||
const integration = integrations.byHost(providerConfig.host);
|
||||
|
||||
if (!providerConfig.orgEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!integration) {
|
||||
throw new Error(
|
||||
`No gitlab integration found that matches host ${providerConfig.host}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!options.schedule && !providerConfig.schedule) {
|
||||
throw new Error(
|
||||
`No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const taskRunner =
|
||||
options.schedule ??
|
||||
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
|
||||
|
||||
providers.push(
|
||||
new GitlabOrgDiscoveryEntityProvider({
|
||||
...options,
|
||||
config: providerConfig,
|
||||
integration,
|
||||
taskRunner,
|
||||
}),
|
||||
);
|
||||
});
|
||||
return providers;
|
||||
}
|
||||
|
||||
private constructor(options: {
|
||||
config: GitlabProviderConfig;
|
||||
integration: GitLabIntegration;
|
||||
logger: Logger;
|
||||
taskRunner: TaskRunner;
|
||||
}) {
|
||||
this.config = options.config;
|
||||
this.integration = options.integration;
|
||||
this.logger = options.logger.child({
|
||||
target: this.getProviderName(),
|
||||
});
|
||||
this.scheduleFn = this.createScheduleFn(options.taskRunner);
|
||||
}
|
||||
|
||||
getProviderName(): string {
|
||||
return `GitlabOrgDiscoveryEntityProvider:${this.config.id}`;
|
||||
}
|
||||
|
||||
async connect(connection: EntityProviderConnection): Promise<void> {
|
||||
this.connection = connection;
|
||||
await this.scheduleFn();
|
||||
}
|
||||
|
||||
private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
|
||||
return async () => {
|
||||
const taskId = `${this.getProviderName()}:refresh`;
|
||||
return taskRunner.run({
|
||||
id: taskId,
|
||||
fn: async () => {
|
||||
const logger = this.logger.child({
|
||||
class: GitlabOrgDiscoveryEntityProvider.prototype.constructor.name,
|
||||
taskId,
|
||||
taskInstanceId: uuid.v4(),
|
||||
});
|
||||
|
||||
try {
|
||||
await this.refresh(logger);
|
||||
} catch (error) {
|
||||
logger.error(`${this.getProviderName()} refresh failed`, error);
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
private async refresh(logger: Logger): Promise<void> {
|
||||
if (!this.connection) {
|
||||
throw new Error(
|
||||
`Gitlab discovery connection not initialized for ${this.getProviderName()}`,
|
||||
);
|
||||
}
|
||||
|
||||
const client = new GitLabClient({
|
||||
config: this.integration.config,
|
||||
logger: logger,
|
||||
});
|
||||
|
||||
const users = paginated<GitLabUser>(options => client.listUsers(options), {
|
||||
page: 1,
|
||||
per_page: 100,
|
||||
active: true,
|
||||
});
|
||||
|
||||
const groups = paginated<GitLabGroup>(
|
||||
options => client.listGroups(options),
|
||||
{
|
||||
page: 1,
|
||||
per_page: 100,
|
||||
},
|
||||
);
|
||||
|
||||
const idMappedGroup: { [groupId: number]: GitLabGroup } = {};
|
||||
|
||||
const res: Result = {
|
||||
scanned: 0,
|
||||
matches: [],
|
||||
};
|
||||
|
||||
const groupRes: GroupResult = {
|
||||
scanned: 0,
|
||||
matches: [],
|
||||
};
|
||||
|
||||
for await (const group of groups) {
|
||||
if (!this.config.groupPattern.test(group.full_path ?? '')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
groupRes.scanned++;
|
||||
groupRes.matches.push(group);
|
||||
|
||||
idMappedGroup[group.id] = group;
|
||||
}
|
||||
|
||||
for await (const user of users) {
|
||||
if (!this.config.userPattern.test(user.email ?? user.username ?? '')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
res.scanned++;
|
||||
|
||||
if (user.state !== 'active') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const memberships = await client.getUserMemberships(user.id);
|
||||
const userGroups: GitLabGroup[] = [];
|
||||
|
||||
for (const i of memberships) {
|
||||
if (
|
||||
i.source_type === 'Namespace' &&
|
||||
idMappedGroup.hasOwnProperty(i.source_id)
|
||||
) {
|
||||
userGroups.push(idMappedGroup[i.source_id]);
|
||||
}
|
||||
}
|
||||
|
||||
user.groups = userGroups;
|
||||
|
||||
res.matches.push(user);
|
||||
}
|
||||
|
||||
const groupsWithUsers = groupRes.matches.filter(group => {
|
||||
return (
|
||||
res.matches.filter(x => {
|
||||
return !!x.groups?.find(y => y.id === group.id);
|
||||
}).length > 0
|
||||
);
|
||||
});
|
||||
|
||||
const userEntities = res.matches.map(p =>
|
||||
this.createUserEntity(p, this.integration.config.host),
|
||||
);
|
||||
const groupEntities = this.createGroupEntities(
|
||||
groupsWithUsers,
|
||||
this.integration.config.host,
|
||||
);
|
||||
|
||||
await this.connection.applyMutation({
|
||||
type: 'full',
|
||||
entities: [...userEntities, ...groupEntities].map(entity => ({
|
||||
locationKey: this.getProviderName(),
|
||||
entity: this.withLocations(this.integration.config.host, entity),
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
private createGroupEntities(
|
||||
groupResult: GitLabGroup[],
|
||||
host: string,
|
||||
): GroupEntity[] {
|
||||
const idMapped: { [groupId: number]: GitLabGroup } = {};
|
||||
const entities: GroupEntity[] = [];
|
||||
|
||||
for (const group of groupResult) {
|
||||
idMapped[group.id] = group;
|
||||
}
|
||||
|
||||
for (const group of groupResult) {
|
||||
const entity = this.createGroupEntity(group, host);
|
||||
|
||||
if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) {
|
||||
entity.spec.parent = idMapped[group.parent_id].full_path;
|
||||
}
|
||||
|
||||
entities.push(entity);
|
||||
}
|
||||
|
||||
return entities;
|
||||
}
|
||||
|
||||
private withLocations(host: string, entity: Entity): Entity {
|
||||
const location =
|
||||
entity.kind === 'Group'
|
||||
? `url:${host}/teams/${entity.metadata.name}`
|
||||
: `url:${host}/${entity.metadata.name}`;
|
||||
return merge(
|
||||
{
|
||||
metadata: {
|
||||
annotations: {
|
||||
[ANNOTATION_LOCATION]: location,
|
||||
[ANNOTATION_ORIGIN_LOCATION]: location,
|
||||
},
|
||||
},
|
||||
},
|
||||
entity,
|
||||
) as Entity;
|
||||
}
|
||||
|
||||
private createUserEntity(user: GitLabUser, host: string): UserEntity {
|
||||
const annotations: { [annotationName: string]: string } = {};
|
||||
|
||||
annotations[`${host}/user-login`] = user.web_url;
|
||||
|
||||
const entity: UserEntity = {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
name: user.username,
|
||||
annotations: annotations,
|
||||
},
|
||||
spec: {
|
||||
profile: {
|
||||
displayName: user.name,
|
||||
picture: user.avatar_url,
|
||||
},
|
||||
memberOf: [],
|
||||
},
|
||||
};
|
||||
|
||||
if (user.email) {
|
||||
if (!entity.spec) {
|
||||
entity.spec = {};
|
||||
}
|
||||
|
||||
if (!entity.spec.profile) {
|
||||
entity.spec.profile = {};
|
||||
}
|
||||
|
||||
entity.spec.profile.email = user.email;
|
||||
}
|
||||
|
||||
if (user.groups) {
|
||||
for (const group of user.groups) {
|
||||
if (!entity.spec.memberOf) {
|
||||
entity.spec.memberOf = [];
|
||||
}
|
||||
entity.spec.memberOf.push(group.full_path.replace('/', '-'));
|
||||
}
|
||||
}
|
||||
|
||||
return entity;
|
||||
}
|
||||
|
||||
private createGroupEntity(group: GitLabGroup, host: string): GroupEntity {
|
||||
const annotations: { [annotationName: string]: string } = {};
|
||||
|
||||
annotations[`${host}/team-path`] = group.full_path;
|
||||
|
||||
const entity: GroupEntity = {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Group',
|
||||
metadata: {
|
||||
name: group.full_path.replace('/', '-'),
|
||||
annotations: annotations,
|
||||
},
|
||||
spec: {
|
||||
type: 'team',
|
||||
children: [],
|
||||
profile: {
|
||||
displayName: group.name,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
if (group.description) {
|
||||
entity.metadata.description = group.description;
|
||||
}
|
||||
|
||||
return entity;
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,9 @@ describe('config', () => {
|
||||
host: 'host',
|
||||
catalogFile: 'catalog-info.yaml',
|
||||
projectPattern: /[\s\S]*/,
|
||||
groupPattern: /[\s\S]*/,
|
||||
userPattern: /[\s\S]*/,
|
||||
orgEnabled: false,
|
||||
schedule: undefined,
|
||||
}),
|
||||
);
|
||||
@@ -85,6 +88,9 @@ describe('config', () => {
|
||||
host: 'host',
|
||||
catalogFile: 'custom-file.yaml',
|
||||
projectPattern: /[\s\S]*/,
|
||||
groupPattern: /[\s\S]*/,
|
||||
userPattern: /[\s\S]*/,
|
||||
orgEnabled: false,
|
||||
schedule: undefined,
|
||||
}),
|
||||
);
|
||||
@@ -120,6 +126,9 @@ describe('config', () => {
|
||||
host: 'host',
|
||||
catalogFile: 'catalog-info.yaml',
|
||||
projectPattern: /[\s\S]*/,
|
||||
groupPattern: /[\s\S]*/,
|
||||
userPattern: /[\s\S]*/,
|
||||
orgEnabled: false,
|
||||
schedule: {
|
||||
frequency: Duration.fromISO('PT30M'),
|
||||
timeout: {
|
||||
|
||||
@@ -35,6 +35,13 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
|
||||
const projectPattern = new RegExp(
|
||||
config.getOptionalString('projectPattern') ?? /[\s\S]*/,
|
||||
);
|
||||
const userPattern = new RegExp(
|
||||
config.getOptionalString('userPattern') ?? /[\s\S]*/,
|
||||
);
|
||||
const groupPattern = new RegExp(
|
||||
config.getOptionalString('groupPattern') ?? /[\s\S]*/,
|
||||
);
|
||||
const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false;
|
||||
|
||||
const schedule = config.has('schedule')
|
||||
? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
|
||||
@@ -47,7 +54,10 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
|
||||
host,
|
||||
catalogFile,
|
||||
projectPattern,
|
||||
userPattern,
|
||||
groupPattern,
|
||||
schedule,
|
||||
orgEnabled,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -15,3 +15,4 @@
|
||||
*/
|
||||
|
||||
export { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider';
|
||||
export { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider';
|
||||
|
||||
Reference in New Issue
Block a user