Merge pull request #29370 from vaceal/master

added support for multiple group patterns instead of a single one

.changeset/chubby-results-relax.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': patch
+---
+
+Added support for multiple group patterns instead of a single one to increase flexibility when filtering groups from GitLab.

docs/integrations/gitlab/discovery.md

@@ -151,8 +151,11 @@ catalog:
         skipForkedRepos: false # Optional. If the project is a fork, skip repository
         includeArchivedRepos: false # Optional. If project is archived, include repository
         group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned
+        groupPattern: # Optional. Filters for groups based on a list of RegEx. Default, no filters.
+          - '^somegroup$'
+          - 'anothergroup'
         entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
-        projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything
+        projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything
         excludeRepos: [] # Optional. A list of project paths that should be excluded from discovery, e.g. group/subgroup/repo. Should not start or end with a slash.
         schedule: # Same options as in SchedulerServiceTaskScheduleDefinition. Optional for the Legacy Backend System
           # supports cron, ISO duration, "human duration" as used in code

plugins/catalog-backend-module-gitlab/config.d.ts

@@ -72,7 +72,7 @@ export interface Config {
           /**
            * (Optional) RegExp for the Group Name Pattern
            */
-          groupPattern?: string;
+          groupPattern?: string | string[];
           /**
            * Specifies the types of group membership relations that should be included when ingesting data.
            *

plugins/catalog-backend-module-gitlab/report.api.md

@@ -111,7 +111,7 @@ export type GitlabProviderConfig = {
   catalogFile: string;
   projectPattern: RegExp;
   userPattern: RegExp;
-  groupPattern: RegExp;
+  groupPattern: RegExp | RegExp[];
   allowInherited?: boolean;
   relations?: string[];
   orgEnabled?: boolean;

plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts

@@ -81,6 +81,13 @@ const httpHandlers = [
     );
   }),
 
+  rest.get(`${apiBaseUrl}/groups/group1`, (_, res, ctx) => {
+    return res(
+      ctx.set('x-next-page', ''),
+      ctx.json(all_groups_response.find(g => g.full_path === 'group1')),
+    );
+  }),
+
   rest.get(`${apiBaseUrl}/groups/42`, (_, res, ctx) => {
     return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' }));
   }),
@@ -229,6 +236,26 @@ const httpGroupListDescendantProjectsByName = all_groups_response.map(group => {
     },
   );
 });
+
+const httpGroupListDescendantProjectsByFullPath = all_groups_response.map(
+  group => {
+    return rest.get(
+      `${apiBaseUrl}/groups/${encodeURIComponent(group.full_path)}/projects`,
+      (req, res, ctx) => {
+        const archived = req.url.searchParams.get('archived');
+
+        const projectsInGroup = all_projects_response.filter(
+          p =>
+            p.path_with_namespace?.includes(group.full_path) &&
+            (archived === 'false' ? !p.archived : true),
+        );
+
+        return res(ctx.json(projectsInGroup));
+      },
+    );
+  },
+);
+
 const httpGroupFindByNameDynamic = all_groups_response.map(group => {
   return rest.get(`${apiBaseUrl}/groups/${group.name}`, (_, res, ctx) => {
     return res(ctx.json(all_groups_response.find(g => g.name === group.name)));
@@ -711,6 +738,7 @@ export const handlers = [
   ...httpGroupFindByNameDynamic,
   ...httpGroupListDescendantProjectsById,
   ...httpGroupListDescendantProjectsByName,
+  ...httpGroupListDescendantProjectsByFullPath,
   ...graphqlHandlers,
   ...httpGroupFindByEncodedPathDynamic,
 ];

plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts

@@ -820,6 +820,95 @@ export const config_org_group_with_subgroups_sass = {
     },
   },
 };
+
+export const config_groupPatterns_only_noMatch = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'example.com',
+        apiBaseUrl: 'https://example.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'example.com',
+          groupPattern: ['^group8$', '^group9$', '^group10$'],
+        },
+      },
+    },
+  },
+};
+
+export const config_groupPatterns_only_Match1Group = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'example.com',
+        apiBaseUrl: 'https://example.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'example.com',
+          groupPattern: ['^group1$', '^group9$', '^group10$'],
+        },
+      },
+    },
+  },
+};
+
+export const config_groupPatterns_multiple_matches = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'example.com',
+        apiBaseUrl: 'https://example.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'example.com',
+          groupPattern: ['^group1$', '^group2$', '^group10$'],
+        },
+      },
+    },
+  },
+};
+
+export const config_groupPatterns_duplicate_match = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'example.com',
+        apiBaseUrl: 'https://example.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'example.com',
+          groupPattern: ['^group1$', 'subgroup1'], // Both patterns match the same group
+        },
+      },
+    },
+  },
+};
+
 /**
  * GitLab API responses
  */
@@ -919,6 +1008,18 @@ export const all_projects_response: GitLabProject[] = [
     web_url: 'https://example.com/group1/test-repo8-archived',
     path_with_namespace: 'group1/test-repo8-archived',
   },
+  // project in subgroup
+  {
+    id: 9,
+    description: 'Project Nine Description',
+    name: 'test-repo9',
+    default_branch: 'main',
+    path: 'test-repo9',
+    archived: false,
+    last_activity_at: new Date().toString(),
+    web_url: 'https://example.com/group1/subgroup1/test-repo9',
+    path_with_namespace: 'group1/subgroup1/test-repo9',
+  },
 ];
 
 export const all_users_response: GitLabUser[] = [
@@ -1211,6 +1312,12 @@ export const all_groups_response: GitLabGroup[] = [
     description: '',
     full_path: 'group1/subgroup2',
   },
+  {
+    id: 8,
+    name: 'awsome-group',
+    description: '',
+    full_path: 'awsome-group',
+  },
 ];
 
 export const group_with_subgroups_response: GitLabGroup[] = [

plugins/catalog-backend-module-gitlab/src/lib/types.ts

@@ -184,11 +184,13 @@ export type GitlabProviderConfig = {
    * Defaults to `[\s\S]*`, which means to not filter anything
    */
   userPattern: RegExp;
+
   /**
-   * Filters found groups based on provided patter.
+   * Filters found groups based on provided patterns.
    * Defaults to `[\s\S]*`, which means to not filter anything
+   *
    */
-  groupPattern: RegExp;
+  groupPattern: RegExp | RegExp[];
 
   /**
    * If true, the provider will also add inherited (ascendant) users to the ingested groups.

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts

@@ -557,5 +557,121 @@ describe('GitlabDiscoveryEntityProvider - events', () => {
     });
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
   });
-  // EventSupportChange: stop add tests >>>
+
+  it('should ignore projects when none of the groups regex patterns match', async () => {
+    const config = new ConfigReader(mock.config_groupPatterns_only_noMatch);
+
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: [],
+    });
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // No entities should be applied
+  });
+
+  it('should include only the project that matches one of the group regex patterns', async () => {
+    const config = new ConfigReader(mock.config_groupPatterns_only_Match1Group);
+
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: mock.expected_location_entities_default_branch.filter(
+        entity =>
+          entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('/group1/'), // Only projects in 'group2' should match
+      ),
+    });
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // Only one matching project should be applied
+  });
+
+  it('should include projects that match multiple group regex patterns', async () => {
+    const config = new ConfigReader(mock.config_groupPatterns_multiple_matches);
+
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: mock.expected_location_entities_default_branch.filter(
+        entity =>
+          entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('/group1/') ||
+          entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('/group2/'),
+      ),
+    });
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // Projects from both groups should be applied
+  });
+
+  it('should not create duplicate locations when multiple groupPatterns match the same group', async () => {
+    const config = new ConfigReader(mock.config_groupPatterns_duplicate_match);
+
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: mock.expected_location_entities_default_branch.filter(entity =>
+        entity.entity.metadata.annotations[
+          'backstage.io/managed-by-location'
+        ].includes('/group1/'),
+      ),
+    });
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+  });
 });

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts

@@ -33,6 +33,7 @@ import { WebhookProjectSchema, WebhookPushEventSchema } from '@gitbeaker/rest';
 import * as uuid from 'uuid';
 import {
   GitLabClient,
+  GitLabGroup,
   GitLabProject,
   GitlabProviderConfig,
   paginated,
@@ -207,10 +208,119 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       );
     }
 
+    const locations = await this.getEntities();
+
+    await this.connection.applyMutation({
+      type: 'full',
+      entities: locations.map(location => ({
+        locationKey: this.getProviderName(),
+        entity: locationSpecToLocationEntity({ location }),
+      })),
+    });
+
+    logger.info(`Processed ${locations.length} locations`);
+  }
+
+  /**
+   * Determine the location on GitLab to be ingested base on configured groups and filters.
+   *
+   * @returns A list of location to be ingested
+   */
+  private async getEntities() {
+    let res: Result = {
+      scanned: 0,
+      matches: [],
+    };
+
+    const groupToProcess = new Map<string, GitLabGroup>();
+    let groupFilters;
+
+    if (this.config.groupPattern !== undefined) {
+      const patterns = Array.isArray(this.config.groupPattern)
+        ? this.config.groupPattern
+        : [this.config.groupPattern];
+
+      if (patterns.length === 1 && patterns[0].source === '[\\s\\S]*') {
+        // if the pattern is a catch-all, we don't need to filter groups
+        groupFilters = new Array<RegExp>();
+      } else {
+        groupFilters = patterns;
+      }
+    }
+
+    if (groupFilters && groupFilters.length > 0) {
+      const groups = paginated<GitLabGroup>(
+        options => this.gitLabClient.listGroups(options),
+        {
+          page: 1,
+          per_page: 50,
+        },
+      );
+
+      for await (const group of groups) {
+        if (
+          groupFilters.some(groupFilter => groupFilter.test(group.full_path)) &&
+          !groupToProcess.has(group.full_path)
+        ) {
+          groupToProcess.set(group.full_path, group);
+        }
+      }
+
+      for (const group of groupToProcess.values()) {
+        const tmpRes = await this.getProjectsToProcess(group.full_path);
+        res.scanned += tmpRes.scanned;
+        // merge both arrays safely
+        for (const project of tmpRes.matches) {
+          res.matches.push(project);
+        }
+      }
+    } else {
+      res = await this.getProjectsToProcess(this.config.group);
+    }
+
+    const locations = this.deduplicateProjects(res.matches).map(p =>
+      this.createLocationSpec(p),
+    );
+
+    this.logger.info(
+      `Processed ${locations.length} from scanned ${res.scanned} projects.`,
+    );
+
+    return locations;
+  }
+
+  /**
+   * Deduplicate a list of projects based on their id.
+   *
+   * @param projects - a list of projects to be deduplicated
+   * @returns a list of projects with unique id
+   */
+  private deduplicateProjects(projects: GitLabProject[]): GitLabProject[] {
+    const uniqueProjects = new Map<number, GitLabProject>();
+
+    for (const project of projects) {
+      uniqueProjects.set(project.id, project);
+    }
+
+    return Array.from(uniqueProjects.values());
+  }
+
+  /**
+   * Retrieve a list of projects that match configuration.
+   *
+   * @param group - a full path of a GitLab group, can be empty
+   * @returns An array of project to be processed and the number of project scanned
+   */
+  private async getProjectsToProcess(group: string) {
+    const res: Result = {
+      scanned: 0,
+      matches: [],
+    };
+
     const projects = paginated<GitLabProject>(
       options => this.gitLabClient.listProjects(options),
       {
-        group: this.config.group,
+        group: group,
         page: 1,
         per_page: 50,
         ...(!this.config.includeArchivedRepos && { archived: false }),
@@ -219,30 +329,15 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       },
     );
 
-    const res: Result = {
-      scanned: 0,
-      matches: [],
-    };
-
     for await (const project of projects) {
+      res.scanned++;
+
       if (await this.shouldProcessProject(project, this.gitLabClient)) {
-        res.scanned++;
         res.matches.push(project);
       }
     }
 
-    const locations = res.matches.map(p => this.createLocationSpec(p));
-
-    logger.info(
-      `Processed ${locations.length} from scanned ${res.scanned} projects.`,
-    );
-    await this.connection.applyMutation({
-      type: 'full',
-      entities: locations.map(location => ({
-        locationKey: this.getProviderName(),
-        entity: locationSpecToLocationEntity({ location }),
-      })),
-    });
+    return res;
   }
 
   private createLocationSpec(project: GitLabProject): LocationSpec {

plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts

@@ -111,6 +111,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
   private groupEntitiesTransformer: GroupEntitiesTransformer;
   private groupNameTransformer: GroupNameTransformer;
   private readonly gitLabClient: GitLabClient;
+  private readonly groupPatterns: RegExp[];
 
   static fromConfig(
     config: Config,
@@ -198,6 +199,10 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     this.groupNameTransformer =
       options.groupNameTransformer ?? defaultGroupNameTransformer;
 
+    this.groupPatterns = Array.isArray(this.config.groupPattern)
+      ? this.config.groupPattern
+      : [this.config.groupPattern];
+
     this.gitLabClient = new GitLabClient({
       config: this.integration.config,
       logger: this.logger,
@@ -464,7 +469,6 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
 
     for await (const group of groups) {
       groupRes.scanned++;
-
       if (!this.shouldProcessGroup(group)) {
         logger.debug(`Skipped group: ${group.full_path}`);
         continue;
@@ -798,7 +802,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
 
   private shouldProcessGroup(group: GitLabGroup): boolean {
     return (
-      this.config.groupPattern.test(group.full_path) &&
+      this.groupPatterns.some(pattern => pattern.test(group.full_path)) &&
       (!this.config.group ||
         group.full_path.startsWith(`${this.config.group}/`) ||
         group.full_path === this.config.group)

plugins/catalog-backend-module-gitlab/src/providers/config.ts

@@ -39,9 +39,21 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
   const userPattern = new RegExp(
     config.getOptionalString('userPattern') ?? /[\s\S]*/,
   );
-  const groupPattern = new RegExp(
-    config.getOptionalString('groupPattern') ?? /[\s\S]*/,
-  );
+
+  const configValue = config.getOptional('groupPattern');
+  let groupPattern;
+
+  if ((configValue && typeof configValue === 'string') || !configValue) {
+    groupPattern = new RegExp(
+      config.getOptionalString('groupPattern') ?? /[\s\S]*/,
+    );
+  } else if (configValue && Array.isArray(configValue)) {
+    const configPattern = config.getOptionalStringArray('groupPattern') ?? [];
+    groupPattern = configPattern.map(pattern => new RegExp(pattern));
+  } else {
+    groupPattern = new RegExp(/[\s\S]*/);
+  }
+
   const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false;
   const allowInherited: boolean =
     config.getOptionalBoolean('allowInherited') ?? false;