diff --git a/.changeset/neat-hotels-wink.md b/.changeset/neat-hotels-wink.md new file mode 100644 index 0000000000..fca82e38fa --- /dev/null +++ b/.changeset/neat-hotels-wink.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Added the option to provide custom `groupTransformer`, `userTransformer` and `groupNameTransformer` to allow custom transformations of groups and users diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index 6620f6445f..bb52d8908a 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -8,10 +8,14 @@ import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { GitLabIntegrationConfig } from '@backstage/integration'; +import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/plugin-catalog-node'; import { Logger } from 'winston'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { UserEntity } from '@backstage/catalog-model'; // @public export class GitlabDiscoveryEntityProvider implements EntityProvider { @@ -53,6 +57,20 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { ): Promise; } +// @public +export type GitLabGroup = { + id: number; + name: string; + full_path: string; + description?: string; + parent_id?: number; +}; + +// @public (undocumented) +export type GitLabGroupSamlIdentity = { + extern_uid: string; +}; + // @public export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { // (undocumented) @@ -64,9 +82,84 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; + userTransformer?: UserTransformer; + groupEntitiesTransformer?: GroupTransformer; + groupNameTransformer?: GroupNameTransformer; }, ): GitlabOrgDiscoveryEntityProvider[]; // (undocumented) getProviderName(): string; } + +// @public +export type GitlabProviderConfig = { + host: string; + group: string; + id: string; + branch?: string; + fallbackBranch: string; + catalogFile: string; + projectPattern: RegExp; + userPattern: RegExp; + groupPattern: RegExp; + orgEnabled?: boolean; + schedule?: TaskScheduleDefinition; + skipForkedRepos?: boolean; +}; + +// @public +export type GitLabUser = { + id: number; + username: string; + email?: string; + name: string; + state: string; + web_url: string; + avatar_url: string; + groups?: GitLabGroup[]; + group_saml_identity?: GitLabGroupSamlIdentity; +}; + +// @public +export type GroupNameTransformer = ( + options: GroupNameTransformerOptions, +) => string; + +// @public +export interface GroupNameTransformerOptions { + // (undocumented) + group: GitLabGroup; + // (undocumented) + providerConfig: GitlabProviderConfig; +} + +// @public +export type GroupTransformer = ( + options: GroupTransformerOptions, +) => GroupEntity[]; + +// @public +export interface GroupTransformerOptions { + // (undocumented) + groupNameTransformer: GroupNameTransformer; + // (undocumented) + groups: GitLabGroup[]; + // (undocumented) + providerConfig: GitlabProviderConfig; +} + +// @public +export type UserTransformer = (options: UserTransformerOptions) => UserEntity; + +// @public +export interface UserTransformerOptions { + // (undocumented) + groupNameTransformer: GroupNameTransformer; + // (undocumented) + integrationConfig: GitLabIntegrationConfig; + // (undocumented) + providerConfig: GitlabProviderConfig; + // (undocumented) + user: GitLabUser; +} ``` diff --git a/plugins/catalog-backend-module-gitlab/src/index.ts b/plugins/catalog-backend-module-gitlab/src/index.ts index 2dd19c041d..cedcb828d6 100644 --- a/plugins/catalog-backend-module-gitlab/src/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/index.ts @@ -25,3 +25,15 @@ export { GitlabDiscoveryEntityProvider, GitlabOrgDiscoveryEntityProvider, } from './providers'; +export type { + GitLabUser, + GitLabGroup, + GitlabProviderConfig, + GitLabGroupSamlIdentity, + GroupNameTransformer, + GroupNameTransformerOptions, + GroupTransformer, + GroupTransformerOptions, + UserTransformer, + UserTransformerOptions, +} from './lib'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts new file mode 100644 index 0000000000..d7a6bf76e9 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts @@ -0,0 +1,150 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { + GitLabGroup, + GroupNameTransformerOptions, + GroupTransformerOptions, + UserTransformerOptions, +} from './types'; + +export function defaultGroupNameTransformer( + options: GroupNameTransformerOptions, +): string { + if ( + options.providerConfig.group && + options.group.full_path.startsWith(`${options.providerConfig.group}/`) + ) { + return options.group.full_path + .replace(`${options.providerConfig.group}/`, '') + .replaceAll('/', '-'); + } + return options.group.full_path.replaceAll('/', '-'); +} + +export function defaultGroupEntitiesTransformer( + options: GroupTransformerOptions, +): GroupEntity[] { + const idMapped: { [groupId: number]: GitLabGroup } = {}; + const entities: GroupEntity[] = []; + + for (const group of options.groups) { + idMapped[group.id] = group; + } + + for (const group of options.groups) { + const annotations: { [annotationName: string]: string } = {}; + + annotations[`${options.providerConfig.host}/team-path`] = group.full_path; + + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: options.groupNameTransformer({ + group, + providerConfig: options.providerConfig, + }), + annotations: annotations, + }, + spec: { + type: 'team', + children: [], + profile: { + displayName: group.name, + }, + }, + }; + + if (group.description) { + entity.metadata.description = group.description; + } + + if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) { + entity.spec.parent = options.groupNameTransformer({ + group: idMapped[group.parent_id], + providerConfig: options.providerConfig, + }); + } + + entities.push(entity); + } + + return entities; +} + +/** + * The default implementation of the transformation from a graph user entry to + * a User entity. + * + * @public + */ +export function defaultUserTransformer( + options: UserTransformerOptions, +): UserEntity { + const annotations: { [annotationName: string]: string } = {}; + + annotations[`${options.integrationConfig.host}/user-login`] = + options.user.web_url; + if (options.user?.group_saml_identity?.extern_uid) { + annotations[`${options.integrationConfig.host}/saml-external-uid`] = + options.user.group_saml_identity.extern_uid; + } + + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: options.user.username, + annotations: annotations, + }, + spec: { + profile: { + displayName: options.user.name || undefined, + picture: options.user.avatar_url || undefined, + }, + memberOf: [], + }, + }; + + if (options.user.email) { + if (!entity.spec) { + entity.spec = {}; + } + + if (!entity.spec.profile) { + entity.spec.profile = {}; + } + + entity.spec.profile.email = options.user.email; + } + + if (options.user.groups) { + for (const group of options.user.groups) { + if (!entity.spec.memberOf) { + entity.spec.memberOf = []; + } + entity.spec.memberOf.push( + options.groupNameTransformer({ + group, + providerConfig: options.providerConfig, + }), + ); + } + } + + return entity; +} diff --git a/plugins/catalog-backend-module-gitlab/src/lib/index.ts b/plugins/catalog-backend-module-gitlab/src/lib/index.ts index 53fad07994..1eee268862 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/index.ts @@ -16,8 +16,17 @@ export { GitLabClient, paginated } from './client'; export type { + GitLabUser, + GitLabGroup, + GitLabGroupSamlIdentity, GitLabProject, GitlabProviderConfig, GitlabGroupDescription, + GroupNameTransformer, + GroupNameTransformerOptions, + GroupTransformer, + GroupTransformerOptions, + UserTransformer, + UserTransformerOptions, } from './types'; export { readGitlabConfigs } from '../providers/config'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index fa2eb73da6..25c9a4a232 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { GitLabIntegrationConfig } from '@backstage/integration'; export type PagedResponse = { items: T[]; @@ -40,6 +42,11 @@ export type GitLabProject = { forked_from_project?: GitlabProjectForkedFrom; }; +/** + * Representation of a GitLab user in the GitLab API + * + * @public + */ export type GitLabUser = { id: number; username: string; @@ -52,10 +59,18 @@ export type GitLabUser = { group_saml_identity?: GitLabGroupSamlIdentity; }; +/** + * @public + */ export type GitLabGroupSamlIdentity = { extern_uid: string; }; +/** + * Representation of a GitLab group in the GitLab API + * + * @public + */ export type GitLabGroup = { id: number; name: string; @@ -113,10 +128,25 @@ export type GitLabDescendantGroupsResponse = { }; }; }; - +/** + * The configuration parameters for the GitlabProvider + * + * @public + */ export type GitlabProviderConfig = { + /** + * Identifies one of the hosts set up in the integrations + */ host: string; + /** + * Required for gitlab.com when `orgEnabled: true`. + * Optional for self managed. Must not end with slash. + * Accepts only groups under the provided path (which will be stripped) + */ group: string; + /** + * ??? + */ id: string; /** * The name of the branch to be used, to discover catalog files. @@ -128,11 +158,85 @@ export type GitlabProviderConfig = { * Defaults to: `master` */ fallbackBranch: string; + /** + * Defaults to `catalog-info.yaml` + */ catalogFile: string; + /** + * Filters found projects based on provided patter. + * Defaults to `[\s\S]*`, which means to not filter anything + */ projectPattern: RegExp; + /** + * Filters found users based on provided patter. + * Defaults to `[\s\S]*`, which means to not filter anything + */ userPattern: RegExp; + /** + * Filters found groups based on provided patter. + * Defaults to `[\s\S]*`, which means to not filter anything + */ groupPattern: RegExp; + orgEnabled?: boolean; schedule?: TaskScheduleDefinition; + /** + * If the project is a fork, skip repository + */ skipForkedRepos?: boolean; }; + +/** + * Customize how group names are generated + * + * @public + */ +export type GroupNameTransformer = ( + options: GroupNameTransformerOptions, +) => string; + +/** + * The GroupTransformerOptions + * + * @public + */ +export interface GroupNameTransformerOptions { + group: GitLabGroup; + providerConfig: GitlabProviderConfig; +} +/** + * Customize the ingested User entity + * + * @public + */ +export type UserTransformer = (options: UserTransformerOptions) => UserEntity; +/** + * The UserTransformerOptions + * + * @public + */ +export interface UserTransformerOptions { + user: GitLabUser; + integrationConfig: GitLabIntegrationConfig; + providerConfig: GitlabProviderConfig; + groupNameTransformer: GroupNameTransformer; +} + +/** + * Customize the ingested Group entity + * + * @public + */ +export type GroupTransformer = ( + options: GroupTransformerOptions, +) => GroupEntity[]; +/** + * The GroupTransformer options + * + * @public + */ +export interface GroupTransformerOptions { + groups: GitLabGroup[]; + providerConfig: GitlabProviderConfig; + groupNameTransformer: GroupNameTransformer; +} diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index affa8a41cf..1157a2f1f3 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -18,8 +18,6 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, Entity, - GroupEntity, - UserEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; @@ -37,7 +35,19 @@ import { paginated, readGitlabConfigs, } from '../lib'; -import { GitLabGroup, GitLabUser, PagedResponse } from '../lib/types'; +import { + GitLabGroup, + GitLabUser, + PagedResponse, + UserTransformer, + GroupTransformer as GroupEntitiesTransformer, + GroupNameTransformer, +} from '../lib/types'; +import { + defaultGroupNameTransformer, + defaultGroupEntitiesTransformer, + defaultUserTransformer, +} from '../lib/defaultTransformers'; type Result = { scanned: number; @@ -59,6 +69,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private readonly logger: Logger; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; + private userTransformer: UserTransformer; + private groupEntitiesTransformer: GroupEntitiesTransformer; + private groupNameTransformer: GroupNameTransformer; static fromConfig( config: Config, @@ -66,6 +79,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; + userTransformer?: UserTransformer; + groupEntitiesTransformer?: GroupEntitiesTransformer; + groupNameTransformer?: GroupNameTransformer; }, ): GitlabOrgDiscoveryEntityProvider[] { if (!options.schedule && !options.scheduler) { @@ -122,6 +138,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { integration: GitLabIntegration; logger: Logger; taskRunner: TaskRunner; + userTransformer?: UserTransformer; + groupEntitiesTransformer?: GroupEntitiesTransformer; + groupNameTransformer?: GroupNameTransformer; }) { this.config = options.config; this.integration = options.integration; @@ -129,6 +148,11 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { target: this.getProviderName(), }); this.scheduleFn = this.createScheduleFn(options.taskRunner); + this.userTransformer = options.userTransformer ?? defaultUserTransformer; + this.groupEntitiesTransformer = + options.groupEntitiesTransformer ?? defaultGroupEntitiesTransformer; + this.groupNameTransformer = + options.groupNameTransformer ?? defaultGroupNameTransformer; } getProviderName(): string { @@ -271,13 +295,20 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); const userEntities = res.matches.map(p => - this.createUserEntity(p, this.integration.config.host), - ); - const groupEntities = this.createGroupEntities( - groupsWithUsers, - this.integration.config.host, + this.userTransformer({ + user: p, + integrationConfig: this.integration.config, + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }), ); + const groupEntities = this.groupEntitiesTransformer({ + groups: groupsWithUsers, + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + await this.connection.applyMutation({ type: 'full', entities: [...userEntities, ...groupEntities].map(entity => ({ @@ -291,32 +322,6 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); } - private createGroupEntities( - groupResult: GitLabGroup[], - host: string, - ): GroupEntity[] { - const idMapped: { [groupId: number]: GitLabGroup } = {}; - const entities: GroupEntity[] = []; - - for (const group of groupResult) { - idMapped[group.id] = group; - } - - for (const group of groupResult) { - const entity = this.createGroupEntity(group, host); - - if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) { - entity.spec.parent = this.groupName( - idMapped[group.parent_id].full_path, - ); - } - - entities.push(entity); - } - - return entities; - } - private withLocations(host: string, baseUrl: string, entity: Entity): Entity { const location = entity.kind === 'Group' @@ -334,90 +339,4 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { entity, ) as Entity; } - - private createUserEntity(user: GitLabUser, host: string): UserEntity { - const annotations: { [annotationName: string]: string } = {}; - - annotations[`${host}/user-login`] = user.web_url; - if (user?.group_saml_identity?.extern_uid) { - annotations[`${host}/saml-external-uid`] = - user.group_saml_identity.extern_uid; - } - - const entity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: user.username, - annotations: annotations, - }, - spec: { - profile: { - displayName: user.name || undefined, - picture: user.avatar_url || undefined, - }, - memberOf: [], - }, - }; - - if (user.email) { - if (!entity.spec) { - entity.spec = {}; - } - - if (!entity.spec.profile) { - entity.spec.profile = {}; - } - - entity.spec.profile.email = user.email; - } - - if (user.groups) { - for (const group of user.groups) { - if (!entity.spec.memberOf) { - entity.spec.memberOf = []; - } - entity.spec.memberOf.push(this.groupName(group.full_path)); - } - } - - return entity; - } - - private groupName(full_path: string): string { - if (this.config.group && full_path.startsWith(`${this.config.group}/`)) { - return full_path - .replace(`${this.config.group}/`, '') - .replaceAll('/', '-'); - } - return full_path.replaceAll('/', '-'); - } - - private createGroupEntity(group: GitLabGroup, host: string): GroupEntity { - const annotations: { [annotationName: string]: string } = {}; - - annotations[`${host}/team-path`] = group.full_path; - - const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: this.groupName(group.full_path), - annotations: annotations, - }, - spec: { - type: 'team', - children: [], - profile: { - displayName: group.name, - }, - }, - }; - - if (group.description) { - entity.metadata.description = group.description; - } - - return entity; - } }