Merge pull request #21903 from markussiebert/catalog-backend-module-gitlab/add-group-and-user-transformers

feat(catalog-backend-module-gitlab): add userTransformer and groupTransformer

.changeset/neat-hotels-wink.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': patch
+---
+
+Added the option to provide custom `groupTransformer`, `userTransformer` and `groupNameTransformer` to allow custom transformations of groups and users

plugins/catalog-backend-module-gitlab/api-report.md

@@ -8,10 +8,14 @@ import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node';
 import { Config } from '@backstage/config';
 import { EntityProvider } from '@backstage/plugin-catalog-node';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
+import { GitLabIntegrationConfig } from '@backstage/integration';
+import { GroupEntity } from '@backstage/catalog-model';
 import { LocationSpec } from '@backstage/plugin-catalog-node';
 import { Logger } from 'winston';
 import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { TaskRunner } from '@backstage/backend-tasks';
+import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+import { UserEntity } from '@backstage/catalog-model';
 
 // @public
 export class GitlabDiscoveryEntityProvider implements EntityProvider {
@@ -53,6 +57,20 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
   ): Promise<boolean>;
 }
 
+// @public
+export type GitLabGroup = {
+  id: number;
+  name: string;
+  full_path: string;
+  description?: string;
+  parent_id?: number;
+};
+
+// @public (undocumented)
+export type GitLabGroupSamlIdentity = {
+  extern_uid: string;
+};
+
 // @public
 export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
   // (undocumented)
@@ -64,9 +82,84 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       logger: Logger;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
+      userTransformer?: UserTransformer;
+      groupEntitiesTransformer?: GroupTransformer;
+      groupNameTransformer?: GroupNameTransformer;
     },
   ): GitlabOrgDiscoveryEntityProvider[];
   // (undocumented)
   getProviderName(): string;
 }
+
+// @public
+export type GitlabProviderConfig = {
+  host: string;
+  group: string;
+  id: string;
+  branch?: string;
+  fallbackBranch: string;
+  catalogFile: string;
+  projectPattern: RegExp;
+  userPattern: RegExp;
+  groupPattern: RegExp;
+  orgEnabled?: boolean;
+  schedule?: TaskScheduleDefinition;
+  skipForkedRepos?: boolean;
+};
+
+// @public
+export type GitLabUser = {
+  id: number;
+  username: string;
+  email?: string;
+  name: string;
+  state: string;
+  web_url: string;
+  avatar_url: string;
+  groups?: GitLabGroup[];
+  group_saml_identity?: GitLabGroupSamlIdentity;
+};
+
+// @public
+export type GroupNameTransformer = (
+  options: GroupNameTransformerOptions,
+) => string;
+
+// @public
+export interface GroupNameTransformerOptions {
+  // (undocumented)
+  group: GitLabGroup;
+  // (undocumented)
+  providerConfig: GitlabProviderConfig;
+}
+
+// @public
+export type GroupTransformer = (
+  options: GroupTransformerOptions,
+) => GroupEntity[];
+
+// @public
+export interface GroupTransformerOptions {
+  // (undocumented)
+  groupNameTransformer: GroupNameTransformer;
+  // (undocumented)
+  groups: GitLabGroup[];
+  // (undocumented)
+  providerConfig: GitlabProviderConfig;
+}
+
+// @public
+export type UserTransformer = (options: UserTransformerOptions) => UserEntity;
+
+// @public
+export interface UserTransformerOptions {
+  // (undocumented)
+  groupNameTransformer: GroupNameTransformer;
+  // (undocumented)
+  integrationConfig: GitLabIntegrationConfig;
+  // (undocumented)
+  providerConfig: GitlabProviderConfig;
+  // (undocumented)
+  user: GitLabUser;
+}
 ```

plugins/catalog-backend-module-gitlab/src/index.ts

@@ -25,3 +25,15 @@ export {
   GitlabDiscoveryEntityProvider,
   GitlabOrgDiscoveryEntityProvider,
 } from './providers';
+export type {
+  GitLabUser,
+  GitLabGroup,
+  GitlabProviderConfig,
+  GitLabGroupSamlIdentity,
+  GroupNameTransformer,
+  GroupNameTransformerOptions,
+  GroupTransformer,
+  GroupTransformerOptions,
+  UserTransformer,
+  UserTransformerOptions,
+} from './lib';

plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts

@@ -0,0 +1,150 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import {
+  GitLabGroup,
+  GroupNameTransformerOptions,
+  GroupTransformerOptions,
+  UserTransformerOptions,
+} from './types';
+
+export function defaultGroupNameTransformer(
+  options: GroupNameTransformerOptions,
+): string {
+  if (
+    options.providerConfig.group &&
+    options.group.full_path.startsWith(`${options.providerConfig.group}/`)
+  ) {
+    return options.group.full_path
+      .replace(`${options.providerConfig.group}/`, '')
+      .replaceAll('/', '-');
+  }
+  return options.group.full_path.replaceAll('/', '-');
+}
+
+export function defaultGroupEntitiesTransformer(
+  options: GroupTransformerOptions,
+): GroupEntity[] {
+  const idMapped: { [groupId: number]: GitLabGroup } = {};
+  const entities: GroupEntity[] = [];
+
+  for (const group of options.groups) {
+    idMapped[group.id] = group;
+  }
+
+  for (const group of options.groups) {
+    const annotations: { [annotationName: string]: string } = {};
+
+    annotations[`${options.providerConfig.host}/team-path`] = group.full_path;
+
+    const entity: GroupEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: options.groupNameTransformer({
+          group,
+          providerConfig: options.providerConfig,
+        }),
+        annotations: annotations,
+      },
+      spec: {
+        type: 'team',
+        children: [],
+        profile: {
+          displayName: group.name,
+        },
+      },
+    };
+
+    if (group.description) {
+      entity.metadata.description = group.description;
+    }
+
+    if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) {
+      entity.spec.parent = options.groupNameTransformer({
+        group: idMapped[group.parent_id],
+        providerConfig: options.providerConfig,
+      });
+    }
+
+    entities.push(entity);
+  }
+
+  return entities;
+}
+
+/**
+ * The default implementation of the transformation from a graph user entry to
+ * a User entity.
+ *
+ * @public
+ */
+export function defaultUserTransformer(
+  options: UserTransformerOptions,
+): UserEntity {
+  const annotations: { [annotationName: string]: string } = {};
+
+  annotations[`${options.integrationConfig.host}/user-login`] =
+    options.user.web_url;
+  if (options.user?.group_saml_identity?.extern_uid) {
+    annotations[`${options.integrationConfig.host}/saml-external-uid`] =
+      options.user.group_saml_identity.extern_uid;
+  }
+
+  const entity: UserEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'User',
+    metadata: {
+      name: options.user.username,
+      annotations: annotations,
+    },
+    spec: {
+      profile: {
+        displayName: options.user.name || undefined,
+        picture: options.user.avatar_url || undefined,
+      },
+      memberOf: [],
+    },
+  };
+
+  if (options.user.email) {
+    if (!entity.spec) {
+      entity.spec = {};
+    }
+
+    if (!entity.spec.profile) {
+      entity.spec.profile = {};
+    }
+
+    entity.spec.profile.email = options.user.email;
+  }
+
+  if (options.user.groups) {
+    for (const group of options.user.groups) {
+      if (!entity.spec.memberOf) {
+        entity.spec.memberOf = [];
+      }
+      entity.spec.memberOf.push(
+        options.groupNameTransformer({
+          group,
+          providerConfig: options.providerConfig,
+        }),
+      );
+    }
+  }
+
+  return entity;
+}

plugins/catalog-backend-module-gitlab/src/lib/index.ts

@@ -16,8 +16,17 @@
 
 export { GitLabClient, paginated } from './client';
 export type {
+  GitLabUser,
+  GitLabGroup,
+  GitLabGroupSamlIdentity,
   GitLabProject,
   GitlabProviderConfig,
   GitlabGroupDescription,
+  GroupNameTransformer,
+  GroupNameTransformerOptions,
+  GroupTransformer,
+  GroupTransformerOptions,
+  UserTransformer,
+  UserTransformerOptions,
 } from './types';
 export { readGitlabConfigs } from '../providers/config';

plugins/catalog-backend-module-gitlab/src/lib/types.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { GitLabIntegrationConfig } from '@backstage/integration';
 
 export type PagedResponse<T> = {
   items: T[];
@@ -40,6 +42,11 @@ export type GitLabProject = {
   forked_from_project?: GitlabProjectForkedFrom;
 };
 
+/**
+ * Representation of a GitLab user in the GitLab API
+ *
+ * @public
+ */
 export type GitLabUser = {
   id: number;
   username: string;
@@ -52,10 +59,18 @@ export type GitLabUser = {
   group_saml_identity?: GitLabGroupSamlIdentity;
 };
 
+/**
+ * @public
+ */
 export type GitLabGroupSamlIdentity = {
   extern_uid: string;
 };
 
+/**
+ * Representation of a GitLab group in the GitLab API
+ *
+ * @public
+ */
 export type GitLabGroup = {
   id: number;
   name: string;
@@ -113,10 +128,25 @@ export type GitLabDescendantGroupsResponse = {
     };
   };
 };
-
+/**
+ * The configuration parameters for the GitlabProvider
+ *
+ * @public
+ */
 export type GitlabProviderConfig = {
+  /**
+   * Identifies one of the hosts set up in the integrations
+   */
   host: string;
+  /**
+   * Required for gitlab.com when `orgEnabled: true`.
+   * Optional for self managed. Must not end with slash.
+   * Accepts only groups under the provided path (which will be stripped)
+   */
   group: string;
+  /**
+   * ???
+   */
   id: string;
   /**
    * The name of the branch to be used, to discover catalog files.
@@ -128,11 +158,85 @@ export type GitlabProviderConfig = {
    * Defaults to: `master`
    */
   fallbackBranch: string;
+  /**
+   * Defaults to `catalog-info.yaml`
+   */
   catalogFile: string;
+  /**
+   * Filters found projects based on provided patter.
+   * Defaults to `[\s\S]*`, which means to not filter anything
+   */
   projectPattern: RegExp;
+  /**
+   * Filters found users based on provided patter.
+   * Defaults to `[\s\S]*`, which means to not filter anything
+   */
   userPattern: RegExp;
+  /**
+   * Filters found groups based on provided patter.
+   * Defaults to `[\s\S]*`, which means to not filter anything
+   */
   groupPattern: RegExp;
+
   orgEnabled?: boolean;
   schedule?: TaskScheduleDefinition;
+  /**
+   * If the project is a fork, skip repository
+   */
   skipForkedRepos?: boolean;
 };
+
+/**
+ * Customize how group names are generated
+ *
+ * @public
+ */
+export type GroupNameTransformer = (
+  options: GroupNameTransformerOptions,
+) => string;
+
+/**
+ * The GroupTransformerOptions
+ *
+ * @public
+ */
+export interface GroupNameTransformerOptions {
+  group: GitLabGroup;
+  providerConfig: GitlabProviderConfig;
+}
+/**
+ * Customize the ingested User entity
+ *
+ * @public
+ */
+export type UserTransformer = (options: UserTransformerOptions) => UserEntity;
+/**
+ * The UserTransformerOptions
+ *
+ * @public
+ */
+export interface UserTransformerOptions {
+  user: GitLabUser;
+  integrationConfig: GitLabIntegrationConfig;
+  providerConfig: GitlabProviderConfig;
+  groupNameTransformer: GroupNameTransformer;
+}
+
+/**
+ * Customize the ingested Group entity
+ *
+ * @public
+ */
+export type GroupTransformer = (
+  options: GroupTransformerOptions,
+) => GroupEntity[];
+/**
+ * The GroupTransformer options
+ *
+ * @public
+ */
+export interface GroupTransformerOptions {
+  groups: GitLabGroup[];
+  providerConfig: GitlabProviderConfig;
+  groupNameTransformer: GroupNameTransformer;
+}

plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts

@@ -18,8 +18,6 @@ import {
   ANNOTATION_LOCATION,
   ANNOTATION_ORIGIN_LOCATION,
   Entity,
-  GroupEntity,
-  UserEntity,
 } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { GitLabIntegration, ScmIntegrations } from '@backstage/integration';
@@ -37,7 +35,19 @@ import {
   paginated,
   readGitlabConfigs,
 } from '../lib';
-import { GitLabGroup, GitLabUser, PagedResponse } from '../lib/types';
+import {
+  GitLabGroup,
+  GitLabUser,
+  PagedResponse,
+  UserTransformer,
+  GroupTransformer as GroupEntitiesTransformer,
+  GroupNameTransformer,
+} from '../lib/types';
+import {
+  defaultGroupNameTransformer,
+  defaultGroupEntitiesTransformer,
+  defaultUserTransformer,
+} from '../lib/defaultTransformers';
 
 type Result = {
   scanned: number;
@@ -59,6 +69,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
   private readonly logger: Logger;
   private readonly scheduleFn: () => Promise<void>;
   private connection?: EntityProviderConnection;
+  private userTransformer: UserTransformer;
+  private groupEntitiesTransformer: GroupEntitiesTransformer;
+  private groupNameTransformer: GroupNameTransformer;
 
   static fromConfig(
     config: Config,
@@ -66,6 +79,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       logger: Logger;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
+      userTransformer?: UserTransformer;
+      groupEntitiesTransformer?: GroupEntitiesTransformer;
+      groupNameTransformer?: GroupNameTransformer;
     },
   ): GitlabOrgDiscoveryEntityProvider[] {
     if (!options.schedule && !options.scheduler) {
@@ -122,6 +138,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     integration: GitLabIntegration;
     logger: Logger;
     taskRunner: TaskRunner;
+    userTransformer?: UserTransformer;
+    groupEntitiesTransformer?: GroupEntitiesTransformer;
+    groupNameTransformer?: GroupNameTransformer;
   }) {
     this.config = options.config;
     this.integration = options.integration;
@@ -129,6 +148,11 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       target: this.getProviderName(),
     });
     this.scheduleFn = this.createScheduleFn(options.taskRunner);
+    this.userTransformer = options.userTransformer ?? defaultUserTransformer;
+    this.groupEntitiesTransformer =
+      options.groupEntitiesTransformer ?? defaultGroupEntitiesTransformer;
+    this.groupNameTransformer =
+      options.groupNameTransformer ?? defaultGroupNameTransformer;
   }
 
   getProviderName(): string {
@@ -271,13 +295,20 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     });
 
     const userEntities = res.matches.map(p =>
-      this.createUserEntity(p, this.integration.config.host),
-    );
-    const groupEntities = this.createGroupEntities(
-      groupsWithUsers,
-      this.integration.config.host,
+      this.userTransformer({
+        user: p,
+        integrationConfig: this.integration.config,
+        providerConfig: this.config,
+        groupNameTransformer: this.groupNameTransformer,
+      }),
     );
 
+    const groupEntities = this.groupEntitiesTransformer({
+      groups: groupsWithUsers,
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+
     await this.connection.applyMutation({
       type: 'full',
       entities: [...userEntities, ...groupEntities].map(entity => ({
@@ -291,32 +322,6 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     });
   }
 
-  private createGroupEntities(
-    groupResult: GitLabGroup[],
-    host: string,
-  ): GroupEntity[] {
-    const idMapped: { [groupId: number]: GitLabGroup } = {};
-    const entities: GroupEntity[] = [];
-
-    for (const group of groupResult) {
-      idMapped[group.id] = group;
-    }
-
-    for (const group of groupResult) {
-      const entity = this.createGroupEntity(group, host);
-
-      if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) {
-        entity.spec.parent = this.groupName(
-          idMapped[group.parent_id].full_path,
-        );
-      }
-
-      entities.push(entity);
-    }
-
-    return entities;
-  }
-
   private withLocations(host: string, baseUrl: string, entity: Entity): Entity {
     const location =
       entity.kind === 'Group'
@@ -334,90 +339,4 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       entity,
     ) as Entity;
   }
-
-  private createUserEntity(user: GitLabUser, host: string): UserEntity {
-    const annotations: { [annotationName: string]: string } = {};
-
-    annotations[`${host}/user-login`] = user.web_url;
-    if (user?.group_saml_identity?.extern_uid) {
-      annotations[`${host}/saml-external-uid`] =
-        user.group_saml_identity.extern_uid;
-    }
-
-    const entity: UserEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'User',
-      metadata: {
-        name: user.username,
-        annotations: annotations,
-      },
-      spec: {
-        profile: {
-          displayName: user.name || undefined,
-          picture: user.avatar_url || undefined,
-        },
-        memberOf: [],
-      },
-    };
-
-    if (user.email) {
-      if (!entity.spec) {
-        entity.spec = {};
-      }
-
-      if (!entity.spec.profile) {
-        entity.spec.profile = {};
-      }
-
-      entity.spec.profile.email = user.email;
-    }
-
-    if (user.groups) {
-      for (const group of user.groups) {
-        if (!entity.spec.memberOf) {
-          entity.spec.memberOf = [];
-        }
-        entity.spec.memberOf.push(this.groupName(group.full_path));
-      }
-    }
-
-    return entity;
-  }
-
-  private groupName(full_path: string): string {
-    if (this.config.group && full_path.startsWith(`${this.config.group}/`)) {
-      return full_path
-        .replace(`${this.config.group}/`, '')
-        .replaceAll('/', '-');
-    }
-    return full_path.replaceAll('/', '-');
-  }
-
-  private createGroupEntity(group: GitLabGroup, host: string): GroupEntity {
-    const annotations: { [annotationName: string]: string } = {};
-
-    annotations[`${host}/team-path`] = group.full_path;
-
-    const entity: GroupEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Group',
-      metadata: {
-        name: this.groupName(group.full_path),
-        annotations: annotations,
-      },
-      spec: {
-        type: 'team',
-        children: [],
-        profile: {
-          displayName: group.name,
-        },
-      },
-    };
-
-    if (group.description) {
-      entity.metadata.description = group.description;
-    }
-
-    return entity;
-  }
 }