From 60e4c2a86d0b67ca47c1c7d2e6d17da7fbb0a235 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 18 Dec 2023 07:48:43 +0100 Subject: [PATCH 1/6] feat: add callback transformers to GitlabOrgDiscoveryEntityProvider Signed-off-by: Markus --- .changeset/neat-hotels-wink.md | 5 + .../src/lib/defaultTransformers.ts | 125 +++++++++++++++ .../src/lib/types.ts | 39 +++++ .../GitlabOrgDiscoveryEntityProvider.ts | 142 ++++++------------ 4 files changed, 211 insertions(+), 100 deletions(-) create mode 100644 .changeset/neat-hotels-wink.md create mode 100644 plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts diff --git a/.changeset/neat-hotels-wink.md b/.changeset/neat-hotels-wink.md new file mode 100644 index 0000000000..a37d282578 --- /dev/null +++ b/.changeset/neat-hotels-wink.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +add `groupTransformer`, `userTransformer` and `groupNameTransformer` to allow custom transformations when groups and users are created diff --git a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts new file mode 100644 index 0000000000..6612086dab --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { + GitLabGroup, + GitLabUser, + GitlabProviderConfig, + GroupNameTransformer, +} from './types'; +import { GitLabIntegrationConfig } from '@backstage/integration'; + +export function defaultGroupNameTransformer( + group: GitLabGroup, + config: GitlabProviderConfig, +): string { + if (config.group && group.full_path.startsWith(`${config.group}/`)) { + return group.full_path.replace(`${config.group}/`, '').replaceAll('/', '-'); + } + return group.full_path.replaceAll('/', '-'); +} + +export function defaultGroupTransformer( + group: GitLabGroup, + provConfig: GitlabProviderConfig, + groupNameTransformer: GroupNameTransformer, +): GroupEntity { + const annotations: { [annotationName: string]: string } = {}; + + annotations[`${provConfig.host}/team-path`] = group.full_path; + + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: groupNameTransformer(group, provConfig), + annotations: annotations, + }, + spec: { + type: 'team', + children: [], + profile: { + displayName: group.name, + }, + }, + }; + + if (group.description) { + entity.metadata.description = group.description; + } + + return entity; +} + +/** + * The default implementation of the transformation from a graph user entry to + * a User entity. + * + * @public + */ +export function defaultUserTransformer( + user: GitLabUser, + intConfig: GitLabIntegrationConfig, + provConfig: GitlabProviderConfig, + groupNameTransformer: GroupNameTransformer, +): UserEntity { + const annotations: { [annotationName: string]: string } = {}; + + annotations[`${intConfig.host}/user-login`] = user.web_url; + if (user?.group_saml_identity?.extern_uid) { + annotations[`${intConfig.host}/saml-external-uid`] = + user.group_saml_identity.extern_uid; + } + + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: user.username, + annotations: annotations, + }, + spec: { + profile: { + displayName: user.name || undefined, + picture: user.avatar_url || undefined, + }, + memberOf: [], + }, + }; + + if (user.email) { + if (!entity.spec) { + entity.spec = {}; + } + + if (!entity.spec.profile) { + entity.spec.profile = {}; + } + + entity.spec.profile.email = user.email; + } + + if (user.groups) { + for (const group of user.groups) { + if (!entity.spec.memberOf) { + entity.spec.memberOf = []; + } + entity.spec.memberOf.push(groupNameTransformer(group, provConfig)); + } + } + + return entity; +} diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index fa2eb73da6..1ecd186a59 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { GitLabIntegrationConfig } from '@backstage/integration'; export type PagedResponse = { items: T[]; @@ -116,6 +118,10 @@ export type GitLabDescendantGroupsResponse = { export type GitlabProviderConfig = { host: string; + /** + * Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned. + * If present this if present, the discovered groups won't contain this prefix + */ group: string; id: string; /** @@ -136,3 +142,36 @@ export type GitlabProviderConfig = { schedule?: TaskScheduleDefinition; skipForkedRepos?: boolean; }; + +/** + * Customize how group names are generated + * + * @public + */ +export type GroupNameTransformer = ( + group: GitLabGroup, + config: GitlabProviderConfig, +) => string; + +/** + * Customize the ingested User entity + * + * @public + */ +export type UserTransformer = ( + user: GitLabUser, + intConfig: GitLabIntegrationConfig, + provConfig: GitlabProviderConfig, + groupNameTransformer: GroupNameTransformer, +) => UserEntity; + +/** + * Customize the ingested Group entity + * + * @public + */ +export type GroupTransformer = ( + group: GitLabGroup, + provConfig: GitlabProviderConfig, + groupNameTransformer: GroupNameTransformer, +) => GroupEntity; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index affa8a41cf..16b1c326f3 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -19,7 +19,6 @@ import { ANNOTATION_ORIGIN_LOCATION, Entity, GroupEntity, - UserEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; @@ -37,7 +36,19 @@ import { paginated, readGitlabConfigs, } from '../lib'; -import { GitLabGroup, GitLabUser, PagedResponse } from '../lib/types'; +import { + GitLabGroup, + GitLabUser, + PagedResponse, + UserTransformer, + GroupTransformer, + GroupNameTransformer, +} from '../lib/types'; +import { + defaultGroupNameTransformer, + defaultGroupTransformer, + defaultUserTransformer, +} from '../lib/defaultTransformers'; type Result = { scanned: number; @@ -59,6 +70,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private readonly logger: Logger; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; + private userTransformer: UserTransformer; + private groupTransformer: GroupTransformer; + private groupNameTransformer: GroupNameTransformer; static fromConfig( config: Config, @@ -66,6 +80,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + groupNameTransformer?: GroupNameTransformer; }, ): GitlabOrgDiscoveryEntityProvider[] { if (!options.schedule && !options.scheduler) { @@ -122,6 +139,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { integration: GitLabIntegration; logger: Logger; taskRunner: TaskRunner; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + groupNameTransformer?: GroupNameTransformer; }) { this.config = options.config; this.integration = options.integration; @@ -129,6 +149,10 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { target: this.getProviderName(), }); this.scheduleFn = this.createScheduleFn(options.taskRunner); + this.userTransformer = options.userTransformer ?? defaultUserTransformer; + this.groupTransformer = options.groupTransformer ?? defaultGroupTransformer; + this.groupNameTransformer = + options.groupNameTransformer ?? defaultGroupNameTransformer; } getProviderName(): string { @@ -271,12 +295,14 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); const userEntities = res.matches.map(p => - this.createUserEntity(p, this.integration.config.host), - ); - const groupEntities = this.createGroupEntities( - groupsWithUsers, - this.integration.config.host, + this.userTransformer( + p, + this.integration.config, + this.config, + this.groupNameTransformer, + ), ); + const groupEntities = this.createGroupEntities(groupsWithUsers); await this.connection.applyMutation({ type: 'full', @@ -291,10 +317,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); } - private createGroupEntities( - groupResult: GitLabGroup[], - host: string, - ): GroupEntity[] { + private createGroupEntities(groupResult: GitLabGroup[]): GroupEntity[] { const idMapped: { [groupId: number]: GitLabGroup } = {}; const entities: GroupEntity[] = []; @@ -303,11 +326,16 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { } for (const group of groupResult) { - const entity = this.createGroupEntity(group, host); + const entity = this.groupTransformer( + group, + this.config, + this.groupNameTransformer, + ); if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) { - entity.spec.parent = this.groupName( - idMapped[group.parent_id].full_path, + entity.spec.parent = this.groupNameTransformer( + idMapped[group.parent_id], + this.config, ); } @@ -334,90 +362,4 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { entity, ) as Entity; } - - private createUserEntity(user: GitLabUser, host: string): UserEntity { - const annotations: { [annotationName: string]: string } = {}; - - annotations[`${host}/user-login`] = user.web_url; - if (user?.group_saml_identity?.extern_uid) { - annotations[`${host}/saml-external-uid`] = - user.group_saml_identity.extern_uid; - } - - const entity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: user.username, - annotations: annotations, - }, - spec: { - profile: { - displayName: user.name || undefined, - picture: user.avatar_url || undefined, - }, - memberOf: [], - }, - }; - - if (user.email) { - if (!entity.spec) { - entity.spec = {}; - } - - if (!entity.spec.profile) { - entity.spec.profile = {}; - } - - entity.spec.profile.email = user.email; - } - - if (user.groups) { - for (const group of user.groups) { - if (!entity.spec.memberOf) { - entity.spec.memberOf = []; - } - entity.spec.memberOf.push(this.groupName(group.full_path)); - } - } - - return entity; - } - - private groupName(full_path: string): string { - if (this.config.group && full_path.startsWith(`${this.config.group}/`)) { - return full_path - .replace(`${this.config.group}/`, '') - .replaceAll('/', '-'); - } - return full_path.replaceAll('/', '-'); - } - - private createGroupEntity(group: GitLabGroup, host: string): GroupEntity { - const annotations: { [annotationName: string]: string } = {}; - - annotations[`${host}/team-path`] = group.full_path; - - const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: this.groupName(group.full_path), - annotations: annotations, - }, - spec: { - type: 'team', - children: [], - profile: { - displayName: group.name, - }, - }, - }; - - if (group.description) { - entity.metadata.description = group.description; - } - - return entity; - } } From 1228c1d584366f4cb9f59a8bd581f6881d2f7e47 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 18 Dec 2023 17:45:33 +0100 Subject: [PATCH 2/6] chore: rename variables Signed-off-by: Markus --- .../src/lib/defaultTransformers.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts index 6612086dab..1fbe60ebe7 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts @@ -34,18 +34,18 @@ export function defaultGroupNameTransformer( export function defaultGroupTransformer( group: GitLabGroup, - provConfig: GitlabProviderConfig, + providerConfig: GitlabProviderConfig, groupNameTransformer: GroupNameTransformer, ): GroupEntity { const annotations: { [annotationName: string]: string } = {}; - annotations[`${provConfig.host}/team-path`] = group.full_path; + annotations[`${providerConfig.host}/team-path`] = group.full_path; const entity: GroupEntity = { apiVersion: 'backstage.io/v1alpha1', kind: 'Group', metadata: { - name: groupNameTransformer(group, provConfig), + name: groupNameTransformer(group, providerConfig), annotations: annotations, }, spec: { @@ -72,15 +72,15 @@ export function defaultGroupTransformer( */ export function defaultUserTransformer( user: GitLabUser, - intConfig: GitLabIntegrationConfig, - provConfig: GitlabProviderConfig, + integrationConfig: GitLabIntegrationConfig, + providerConfig: GitlabProviderConfig, groupNameTransformer: GroupNameTransformer, ): UserEntity { const annotations: { [annotationName: string]: string } = {}; - annotations[`${intConfig.host}/user-login`] = user.web_url; + annotations[`${integrationConfig.host}/user-login`] = user.web_url; if (user?.group_saml_identity?.extern_uid) { - annotations[`${intConfig.host}/saml-external-uid`] = + annotations[`${integrationConfig.host}/saml-external-uid`] = user.group_saml_identity.extern_uid; } @@ -117,7 +117,7 @@ export function defaultUserTransformer( if (!entity.spec.memberOf) { entity.spec.memberOf = []; } - entity.spec.memberOf.push(groupNameTransformer(group, provConfig)); + entity.spec.memberOf.push(groupNameTransformer(group, providerConfig)); } } From 198ef79ee4123fd297f534f673f00e951fd30ab8 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 18 Dec 2023 18:27:30 +0100 Subject: [PATCH 3/6] chore: refactor function inputs Signed-off-by: Markus --- .../src/lib/defaultTransformers.ts | 129 +++++++++++------- .../src/lib/types.ts | 31 +++-- .../GitlabOrgDiscoveryEntityProvider.ts | 61 +++------ 3 files changed, 115 insertions(+), 106 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts index 1fbe60ebe7..d7a6bf76e9 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts @@ -16,52 +16,74 @@ import { GroupEntity, UserEntity } from '@backstage/catalog-model'; import { GitLabGroup, - GitLabUser, - GitlabProviderConfig, - GroupNameTransformer, + GroupNameTransformerOptions, + GroupTransformerOptions, + UserTransformerOptions, } from './types'; -import { GitLabIntegrationConfig } from '@backstage/integration'; export function defaultGroupNameTransformer( - group: GitLabGroup, - config: GitlabProviderConfig, + options: GroupNameTransformerOptions, ): string { - if (config.group && group.full_path.startsWith(`${config.group}/`)) { - return group.full_path.replace(`${config.group}/`, '').replaceAll('/', '-'); + if ( + options.providerConfig.group && + options.group.full_path.startsWith(`${options.providerConfig.group}/`) + ) { + return options.group.full_path + .replace(`${options.providerConfig.group}/`, '') + .replaceAll('/', '-'); } - return group.full_path.replaceAll('/', '-'); + return options.group.full_path.replaceAll('/', '-'); } -export function defaultGroupTransformer( - group: GitLabGroup, - providerConfig: GitlabProviderConfig, - groupNameTransformer: GroupNameTransformer, -): GroupEntity { - const annotations: { [annotationName: string]: string } = {}; +export function defaultGroupEntitiesTransformer( + options: GroupTransformerOptions, +): GroupEntity[] { + const idMapped: { [groupId: number]: GitLabGroup } = {}; + const entities: GroupEntity[] = []; - annotations[`${providerConfig.host}/team-path`] = group.full_path; - - const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: groupNameTransformer(group, providerConfig), - annotations: annotations, - }, - spec: { - type: 'team', - children: [], - profile: { - displayName: group.name, - }, - }, - }; - - if (group.description) { - entity.metadata.description = group.description; + for (const group of options.groups) { + idMapped[group.id] = group; } - return entity; + for (const group of options.groups) { + const annotations: { [annotationName: string]: string } = {}; + + annotations[`${options.providerConfig.host}/team-path`] = group.full_path; + + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: options.groupNameTransformer({ + group, + providerConfig: options.providerConfig, + }), + annotations: annotations, + }, + spec: { + type: 'team', + children: [], + profile: { + displayName: group.name, + }, + }, + }; + + if (group.description) { + entity.metadata.description = group.description; + } + + if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) { + entity.spec.parent = options.groupNameTransformer({ + group: idMapped[group.parent_id], + providerConfig: options.providerConfig, + }); + } + + entities.push(entity); + } + + return entities; } /** @@ -71,36 +93,34 @@ export function defaultGroupTransformer( * @public */ export function defaultUserTransformer( - user: GitLabUser, - integrationConfig: GitLabIntegrationConfig, - providerConfig: GitlabProviderConfig, - groupNameTransformer: GroupNameTransformer, + options: UserTransformerOptions, ): UserEntity { const annotations: { [annotationName: string]: string } = {}; - annotations[`${integrationConfig.host}/user-login`] = user.web_url; - if (user?.group_saml_identity?.extern_uid) { - annotations[`${integrationConfig.host}/saml-external-uid`] = - user.group_saml_identity.extern_uid; + annotations[`${options.integrationConfig.host}/user-login`] = + options.user.web_url; + if (options.user?.group_saml_identity?.extern_uid) { + annotations[`${options.integrationConfig.host}/saml-external-uid`] = + options.user.group_saml_identity.extern_uid; } const entity: UserEntity = { apiVersion: 'backstage.io/v1alpha1', kind: 'User', metadata: { - name: user.username, + name: options.user.username, annotations: annotations, }, spec: { profile: { - displayName: user.name || undefined, - picture: user.avatar_url || undefined, + displayName: options.user.name || undefined, + picture: options.user.avatar_url || undefined, }, memberOf: [], }, }; - if (user.email) { + if (options.user.email) { if (!entity.spec) { entity.spec = {}; } @@ -109,15 +129,20 @@ export function defaultUserTransformer( entity.spec.profile = {}; } - entity.spec.profile.email = user.email; + entity.spec.profile.email = options.user.email; } - if (user.groups) { - for (const group of user.groups) { + if (options.user.groups) { + for (const group of options.user.groups) { if (!entity.spec.memberOf) { entity.spec.memberOf = []; } - entity.spec.memberOf.push(groupNameTransformer(group, providerConfig)); + entity.spec.memberOf.push( + options.groupNameTransformer({ + group, + providerConfig: options.providerConfig, + }), + ); } } diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 1ecd186a59..4a15d44d04 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -149,21 +149,25 @@ export type GitlabProviderConfig = { * @public */ export type GroupNameTransformer = ( - group: GitLabGroup, - config: GitlabProviderConfig, + options: GroupNameTransformerOptions, ) => string; +export interface GroupNameTransformerOptions { + group: GitLabGroup; + providerConfig: GitlabProviderConfig; +} /** * Customize the ingested User entity * * @public */ -export type UserTransformer = ( - user: GitLabUser, - intConfig: GitLabIntegrationConfig, - provConfig: GitlabProviderConfig, - groupNameTransformer: GroupNameTransformer, -) => UserEntity; +export type UserTransformer = (options: UserTransformerOptions) => UserEntity; +export interface UserTransformerOptions { + user: GitLabUser; + integrationConfig: GitLabIntegrationConfig; + providerConfig: GitlabProviderConfig; + groupNameTransformer: GroupNameTransformer; +} /** * Customize the ingested Group entity @@ -171,7 +175,10 @@ export type UserTransformer = ( * @public */ export type GroupTransformer = ( - group: GitLabGroup, - provConfig: GitlabProviderConfig, - groupNameTransformer: GroupNameTransformer, -) => GroupEntity; + options: GroupTransformerOptions, +) => GroupEntity[]; +export interface GroupTransformerOptions { + groups: GitLabGroup[]; + providerConfig: GitlabProviderConfig; + groupNameTransformer: GroupNameTransformer; +} diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 16b1c326f3..1157a2f1f3 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -18,7 +18,6 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, Entity, - GroupEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; @@ -41,12 +40,12 @@ import { GitLabUser, PagedResponse, UserTransformer, - GroupTransformer, + GroupTransformer as GroupEntitiesTransformer, GroupNameTransformer, } from '../lib/types'; import { defaultGroupNameTransformer, - defaultGroupTransformer, + defaultGroupEntitiesTransformer, defaultUserTransformer, } from '../lib/defaultTransformers'; @@ -71,7 +70,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; private userTransformer: UserTransformer; - private groupTransformer: GroupTransformer; + private groupEntitiesTransformer: GroupEntitiesTransformer; private groupNameTransformer: GroupNameTransformer; static fromConfig( @@ -81,7 +80,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { schedule?: TaskRunner; scheduler?: PluginTaskScheduler; userTransformer?: UserTransformer; - groupTransformer?: GroupTransformer; + groupEntitiesTransformer?: GroupEntitiesTransformer; groupNameTransformer?: GroupNameTransformer; }, ): GitlabOrgDiscoveryEntityProvider[] { @@ -140,7 +139,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { logger: Logger; taskRunner: TaskRunner; userTransformer?: UserTransformer; - groupTransformer?: GroupTransformer; + groupEntitiesTransformer?: GroupEntitiesTransformer; groupNameTransformer?: GroupNameTransformer; }) { this.config = options.config; @@ -150,7 +149,8 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); this.scheduleFn = this.createScheduleFn(options.taskRunner); this.userTransformer = options.userTransformer ?? defaultUserTransformer; - this.groupTransformer = options.groupTransformer ?? defaultGroupTransformer; + this.groupEntitiesTransformer = + options.groupEntitiesTransformer ?? defaultGroupEntitiesTransformer; this.groupNameTransformer = options.groupNameTransformer ?? defaultGroupNameTransformer; } @@ -295,14 +295,19 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); const userEntities = res.matches.map(p => - this.userTransformer( - p, - this.integration.config, - this.config, - this.groupNameTransformer, - ), + this.userTransformer({ + user: p, + integrationConfig: this.integration.config, + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }), ); - const groupEntities = this.createGroupEntities(groupsWithUsers); + + const groupEntities = this.groupEntitiesTransformer({ + groups: groupsWithUsers, + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); await this.connection.applyMutation({ type: 'full', @@ -317,34 +322,6 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); } - private createGroupEntities(groupResult: GitLabGroup[]): GroupEntity[] { - const idMapped: { [groupId: number]: GitLabGroup } = {}; - const entities: GroupEntity[] = []; - - for (const group of groupResult) { - idMapped[group.id] = group; - } - - for (const group of groupResult) { - const entity = this.groupTransformer( - group, - this.config, - this.groupNameTransformer, - ); - - if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) { - entity.spec.parent = this.groupNameTransformer( - idMapped[group.parent_id], - this.config, - ); - } - - entities.push(entity); - } - - return entities; - } - private withLocations(host: string, baseUrl: string, entity: Entity): Entity { const location = entity.kind === 'Group' From 08e2eb67b2270a9d2b9be5c7ec541bc0b9de26a6 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 18 Dec 2023 18:30:57 +0100 Subject: [PATCH 4/6] chore: update api report Signed-off-by: Markus --- plugins/catalog-backend-module-gitlab/api-report.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index 6620f6445f..eeab252b59 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -8,10 +8,14 @@ import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { GitLabIntegrationConfig } from '@backstage/integration'; +import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/plugin-catalog-node'; import { Logger } from 'winston'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { UserEntity } from '@backstage/catalog-model'; // @public export class GitlabDiscoveryEntityProvider implements EntityProvider { @@ -64,9 +68,18 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; + userTransformer?: UserTransformer; + groupEntitiesTransformer?: GroupTransformer; + groupNameTransformer?: GroupNameTransformer; }, ): GitlabOrgDiscoveryEntityProvider[]; // (undocumented) getProviderName(): string; } + +// Warnings were encountered during analysis: +// +// src/providers/GitlabOrgDiscoveryEntityProvider.d.ts:23:9 - (ae-forgotten-export) The symbol "UserTransformer" needs to be exported by the entry point index.d.ts +// src/providers/GitlabOrgDiscoveryEntityProvider.d.ts:24:9 - (ae-forgotten-export) The symbol "GroupTransformer" needs to be exported by the entry point index.d.ts +// src/providers/GitlabOrgDiscoveryEntityProvider.d.ts:25:9 - (ae-forgotten-export) The symbol "GroupNameTransformer" needs to be exported by the entry point index.d.ts ``` From 926abed8507cae629560b5503ed232bfc7840342 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 18 Dec 2023 22:06:51 +0100 Subject: [PATCH 5/6] chore: api report Signed-off-by: Markus --- .../api-report.md | 90 +++++++++++++++++-- .../src/index.ts | 12 +++ .../src/lib/index.ts | 9 ++ .../src/lib/types.ts | 64 ++++++++++++- 4 files changed, 167 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index eeab252b59..bb52d8908a 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -57,6 +57,20 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { ): Promise; } +// @public +export type GitLabGroup = { + id: number; + name: string; + full_path: string; + description?: string; + parent_id?: number; +}; + +// @public (undocumented) +export type GitLabGroupSamlIdentity = { + extern_uid: string; +}; + // @public export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { // (undocumented) @@ -77,9 +91,75 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { getProviderName(): string; } -// Warnings were encountered during analysis: -// -// src/providers/GitlabOrgDiscoveryEntityProvider.d.ts:23:9 - (ae-forgotten-export) The symbol "UserTransformer" needs to be exported by the entry point index.d.ts -// src/providers/GitlabOrgDiscoveryEntityProvider.d.ts:24:9 - (ae-forgotten-export) The symbol "GroupTransformer" needs to be exported by the entry point index.d.ts -// src/providers/GitlabOrgDiscoveryEntityProvider.d.ts:25:9 - (ae-forgotten-export) The symbol "GroupNameTransformer" needs to be exported by the entry point index.d.ts +// @public +export type GitlabProviderConfig = { + host: string; + group: string; + id: string; + branch?: string; + fallbackBranch: string; + catalogFile: string; + projectPattern: RegExp; + userPattern: RegExp; + groupPattern: RegExp; + orgEnabled?: boolean; + schedule?: TaskScheduleDefinition; + skipForkedRepos?: boolean; +}; + +// @public +export type GitLabUser = { + id: number; + username: string; + email?: string; + name: string; + state: string; + web_url: string; + avatar_url: string; + groups?: GitLabGroup[]; + group_saml_identity?: GitLabGroupSamlIdentity; +}; + +// @public +export type GroupNameTransformer = ( + options: GroupNameTransformerOptions, +) => string; + +// @public +export interface GroupNameTransformerOptions { + // (undocumented) + group: GitLabGroup; + // (undocumented) + providerConfig: GitlabProviderConfig; +} + +// @public +export type GroupTransformer = ( + options: GroupTransformerOptions, +) => GroupEntity[]; + +// @public +export interface GroupTransformerOptions { + // (undocumented) + groupNameTransformer: GroupNameTransformer; + // (undocumented) + groups: GitLabGroup[]; + // (undocumented) + providerConfig: GitlabProviderConfig; +} + +// @public +export type UserTransformer = (options: UserTransformerOptions) => UserEntity; + +// @public +export interface UserTransformerOptions { + // (undocumented) + groupNameTransformer: GroupNameTransformer; + // (undocumented) + integrationConfig: GitLabIntegrationConfig; + // (undocumented) + providerConfig: GitlabProviderConfig; + // (undocumented) + user: GitLabUser; +} ``` diff --git a/plugins/catalog-backend-module-gitlab/src/index.ts b/plugins/catalog-backend-module-gitlab/src/index.ts index 2dd19c041d..cedcb828d6 100644 --- a/plugins/catalog-backend-module-gitlab/src/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/index.ts @@ -25,3 +25,15 @@ export { GitlabDiscoveryEntityProvider, GitlabOrgDiscoveryEntityProvider, } from './providers'; +export type { + GitLabUser, + GitLabGroup, + GitlabProviderConfig, + GitLabGroupSamlIdentity, + GroupNameTransformer, + GroupNameTransformerOptions, + GroupTransformer, + GroupTransformerOptions, + UserTransformer, + UserTransformerOptions, +} from './lib'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/index.ts b/plugins/catalog-backend-module-gitlab/src/lib/index.ts index 53fad07994..1eee268862 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/index.ts @@ -16,8 +16,17 @@ export { GitLabClient, paginated } from './client'; export type { + GitLabUser, + GitLabGroup, + GitLabGroupSamlIdentity, GitLabProject, GitlabProviderConfig, GitlabGroupDescription, + GroupNameTransformer, + GroupNameTransformerOptions, + GroupTransformer, + GroupTransformerOptions, + UserTransformer, + UserTransformerOptions, } from './types'; export { readGitlabConfigs } from '../providers/config'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 4a15d44d04..25c9a4a232 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -42,6 +42,11 @@ export type GitLabProject = { forked_from_project?: GitlabProjectForkedFrom; }; +/** + * Representation of a GitLab user in the GitLab API + * + * @public + */ export type GitLabUser = { id: number; username: string; @@ -54,10 +59,18 @@ export type GitLabUser = { group_saml_identity?: GitLabGroupSamlIdentity; }; +/** + * @public + */ export type GitLabGroupSamlIdentity = { extern_uid: string; }; +/** + * Representation of a GitLab group in the GitLab API + * + * @public + */ export type GitLabGroup = { id: number; name: string; @@ -115,14 +128,25 @@ export type GitLabDescendantGroupsResponse = { }; }; }; - +/** + * The configuration parameters for the GitlabProvider + * + * @public + */ export type GitlabProviderConfig = { + /** + * Identifies one of the hosts set up in the integrations + */ host: string; /** - * Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned. - * If present this if present, the discovered groups won't contain this prefix + * Required for gitlab.com when `orgEnabled: true`. + * Optional for self managed. Must not end with slash. + * Accepts only groups under the provided path (which will be stripped) */ group: string; + /** + * ??? + */ id: string; /** * The name of the branch to be used, to discover catalog files. @@ -134,12 +158,31 @@ export type GitlabProviderConfig = { * Defaults to: `master` */ fallbackBranch: string; + /** + * Defaults to `catalog-info.yaml` + */ catalogFile: string; + /** + * Filters found projects based on provided patter. + * Defaults to `[\s\S]*`, which means to not filter anything + */ projectPattern: RegExp; + /** + * Filters found users based on provided patter. + * Defaults to `[\s\S]*`, which means to not filter anything + */ userPattern: RegExp; + /** + * Filters found groups based on provided patter. + * Defaults to `[\s\S]*`, which means to not filter anything + */ groupPattern: RegExp; + orgEnabled?: boolean; schedule?: TaskScheduleDefinition; + /** + * If the project is a fork, skip repository + */ skipForkedRepos?: boolean; }; @@ -152,6 +195,11 @@ export type GroupNameTransformer = ( options: GroupNameTransformerOptions, ) => string; +/** + * The GroupTransformerOptions + * + * @public + */ export interface GroupNameTransformerOptions { group: GitLabGroup; providerConfig: GitlabProviderConfig; @@ -162,6 +210,11 @@ export interface GroupNameTransformerOptions { * @public */ export type UserTransformer = (options: UserTransformerOptions) => UserEntity; +/** + * The UserTransformerOptions + * + * @public + */ export interface UserTransformerOptions { user: GitLabUser; integrationConfig: GitLabIntegrationConfig; @@ -177,6 +230,11 @@ export interface UserTransformerOptions { export type GroupTransformer = ( options: GroupTransformerOptions, ) => GroupEntity[]; +/** + * The GroupTransformer options + * + * @public + */ export interface GroupTransformerOptions { groups: GitLabGroup[]; providerConfig: GitlabProviderConfig; From b17facf2f7c874b28d26c69f4a35bcf027609c2c Mon Sep 17 00:00:00 2001 From: Markus Siebert Date: Sat, 23 Dec 2023 08:46:33 +0100 Subject: [PATCH 6/6] neat-hotels-wink.md aktualisieren Co-authored-by: Vincenzo Scamporlino Signed-off-by: Markus Siebert --- .changeset/neat-hotels-wink.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/neat-hotels-wink.md b/.changeset/neat-hotels-wink.md index a37d282578..fca82e38fa 100644 --- a/.changeset/neat-hotels-wink.md +++ b/.changeset/neat-hotels-wink.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend-module-gitlab': patch --- -add `groupTransformer`, `userTransformer` and `groupNameTransformer` to allow custom transformations when groups and users are created +Added the option to provide custom `groupTransformer`, `userTransformer` and `groupNameTransformer` to allow custom transformations of groups and users