added support for multiple group patterns instead of a single one to increase flexibility when filtering groups from GitLab.

Signed-off-by: Alexandre Fournier <vaceal@gmail.com>

.changeset/chubby-results-relax.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': minor
+---
+
+Added support for multiple group patterns instead of a single one to increase flexibility when filtering groups from GitLab.

docs/integrations/gitlab/discovery.md

@@ -151,6 +151,9 @@ catalog:
         skipForkedRepos: false # Optional. If the project is a fork, skip repository
         includeArchivedRepos: false # Optional. If project is archived, include repository
         group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned
+        groupPatterns: # Optional. Filters for groups based on a list of patterns. Default, no filters.
+          - '^somegroup$'
+          - 'anothergroup'
         entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
         projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything
         excludeRepos: [] # Optional. A list of project paths that should be excluded from discovery, e.g. group/subgroup/repo. Should not start or end with a slash.

plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts

@@ -229,6 +229,26 @@ const httpGroupListDescendantProjectsByName = all_groups_response.map(group => {
     },
   );
 });
+
+const httpGroupListDescendantProjectsByFullPath = all_groups_response.map(
+  group => {
+    return rest.get(
+      `${apiBaseUrl}/groups/${encodeURIComponent(group.full_path)}/projects`,
+      (req, res, ctx) => {
+        const archived = req.url.searchParams.get('archived');
+
+        const projectsInGroup = all_projects_response.filter(
+          p =>
+            p.path_with_namespace?.includes(group.full_path) &&
+            (archived === 'false' ? !p.archived : true),
+        );
+
+        return res(ctx.json(projectsInGroup));
+      },
+    );
+  },
+);
+
 const httpGroupFindByNameDynamic = all_groups_response.map(group => {
   return rest.get(`${apiBaseUrl}/groups/${group.name}`, (_, res, ctx) => {
     return res(ctx.json(all_groups_response.find(g => g.name === group.name)));
@@ -711,6 +731,7 @@ export const handlers = [
   ...httpGroupFindByNameDynamic,
   ...httpGroupListDescendantProjectsById,
   ...httpGroupListDescendantProjectsByName,
+  ...httpGroupListDescendantProjectsByFullPath,
   ...graphqlHandlers,
   ...httpGroupFindByEncodedPathDynamic,
 ];

plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts

@@ -820,6 +820,95 @@ export const config_org_group_with_subgroups_sass = {
     },
   },
 };
+
+export const config_groupPatterns_only_noMatch = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'example.com',
+        apiBaseUrl: 'https://example.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'example.com',
+          groupPatterns: ['^group8$', '^group9$', '^group10$'],
+        },
+      },
+    },
+  },
+};
+
+export const config_groupPatterns_only_Match1Group = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'example.com',
+        apiBaseUrl: 'https://example.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'example.com',
+          groupPatterns: ['^group1$', '^group9$', '^group10$'],
+        },
+      },
+    },
+  },
+};
+
+export const config_groupPatterns_multiple_matches = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'example.com',
+        apiBaseUrl: 'https://example.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'example.com',
+          groupPatterns: ['^group1$', '^group2$', '^group10$'],
+        },
+      },
+    },
+  },
+};
+
+export const config_groupPatterns_duplicate_match = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'example.com',
+        apiBaseUrl: 'https://example.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'example.com',
+          groupPatterns: ['^group1$', 'subgroup1'], // Both patterns match the same group
+        },
+      },
+    },
+  },
+};
+
 /**
  * GitLab API responses
  */
@@ -919,6 +1008,18 @@ export const all_projects_response: GitLabProject[] = [
     web_url: 'https://example.com/group1/test-repo8-archived',
     path_with_namespace: 'group1/test-repo8-archived',
   },
+  // project in subgroup
+  {
+    id: 9,
+    description: 'Project Nine Description',
+    name: 'test-repo9',
+    default_branch: 'main',
+    path: 'test-repo9',
+    archived: false,
+    last_activity_at: new Date().toString(),
+    web_url: 'https://example.com/group1/subgroup1/test-repo9',
+    path_with_namespace: 'group1/subgroup1/test-repo9',
+  },
 ];
 
 export const all_users_response: GitLabUser[] = [

plugins/catalog-backend-module-gitlab/src/lib/types.ts

@@ -184,12 +184,20 @@ export type GitlabProviderConfig = {
    * Defaults to `[\s\S]*`, which means to not filter anything
    */
   userPattern: RegExp;
+
   /**
-   * Filters found groups based on provided patter.
+   * Filters found groups based on provided pattern.
    * Defaults to `[\s\S]*`, which means to not filter anything
+   *
+   * @deprecated Use groupPatterns that provide a list instead.
    */
   groupPattern: RegExp;
 
+  /**
+   * Optional, provide a list of pattern for each can match a list of groups.
+   */
+  groupPatterns: RegExp[];
+
   /**
    * If true, the provider will also add inherited (ascendant) users to the ingested groups.
    * See: https://docs.gitlab.com/ee/api/graphql/reference/#groupmemberrelation

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts

@@ -558,4 +558,125 @@ describe('GitlabDiscoveryEntityProvider - events', () => {
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
   });
   // EventSupportChange: stop add tests >>>
+
+  // add the following test:
+  // setup: 3 projects, 2 groups
+  // 1 test where none match --> OK
+  // 1 test where match only 1 group --> OK
+  // 1 test where match 2 groups
+  // 1 test where match 1 group with 2 identical regex
+
+  it('should ignore projects when none of the groups regex patterns match', async () => {
+    const config = new ConfigReader(mock.config_groupPatterns_only_noMatch);
+
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: [],
+    });
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // No entities should be applied
+  });
+});
+it('should include only the project that matches one of the group regex patterns', async () => {
+  const config = new ConfigReader(mock.config_groupPatterns_only_Match1Group);
+
+  const schedule = new PersistingTaskRunner();
+  const entityProviderConnection: EntityProviderConnection = {
+    applyMutation: jest.fn(),
+    refresh: jest.fn(),
+  };
+  const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+    logger,
+    schedule,
+  })[0];
+
+  await provider.connect(entityProviderConnection);
+
+  await provider.refresh(logger);
+
+  expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+    type: 'full',
+    entities: mock.expected_location_entities_default_branch.filter(
+      entity =>
+        entity.entity.metadata.annotations[
+          'backstage.io/managed-by-location'
+        ].includes('/group1/'), // Only projects in 'group2' should match
+    ),
+  });
+
+  expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // Only one matching project should be applied
+});
+it('should include projects that match multiple group regex patterns', async () => {
+  const config = new ConfigReader(mock.config_groupPatterns_multiple_matches);
+
+  const schedule = new PersistingTaskRunner();
+  const entityProviderConnection: EntityProviderConnection = {
+    applyMutation: jest.fn(),
+    refresh: jest.fn(),
+  };
+  const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+    logger,
+    schedule,
+  })[0];
+
+  await provider.connect(entityProviderConnection);
+
+  await provider.refresh(logger);
+
+  expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+    type: 'full',
+    entities: mock.expected_location_entities_default_branch.filter(
+      entity =>
+        entity.entity.metadata.annotations[
+          'backstage.io/managed-by-location'
+        ].includes('/group1/') ||
+        entity.entity.metadata.annotations[
+          'backstage.io/managed-by-location'
+        ].includes('/group2/'),
+    ),
+  });
+
+  expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // Projects from both groups should be applied
+});
+it('should not create duplicate locations when multiple groupPatterns match the same group', async () => {
+  const config = new ConfigReader(mock.config_groupPatterns_duplicate_match);
+
+  const schedule = new PersistingTaskRunner();
+  const entityProviderConnection: EntityProviderConnection = {
+    applyMutation: jest.fn(),
+    refresh: jest.fn(),
+  };
+  const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+    logger,
+    schedule,
+  })[0];
+
+  await provider.connect(entityProviderConnection);
+
+  await provider.refresh(logger);
+
+  expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+    type: 'full',
+    entities: mock.expected_location_entities_default_branch.filter(entity =>
+      entity.entity.metadata.annotations[
+        'backstage.io/managed-by-location'
+      ].includes('/group1/'),
+    ),
+  });
+
+  expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
 });

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts

@@ -33,6 +33,7 @@ import { WebhookProjectSchema, WebhookPushEventSchema } from '@gitbeaker/rest';
 import * as uuid from 'uuid';
 import {
   GitLabClient,
+  GitLabGroup,
   GitLabProject,
   GitlabProviderConfig,
   paginated,
@@ -207,10 +208,107 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       );
     }
 
+    const locations = await this.GetLocations();
+
+    await this.connection.applyMutation({
+      type: 'full',
+      entities: locations.map(location => ({
+        locationKey: this.getProviderName(),
+        entity: locationSpecToLocationEntity({ location }),
+      })),
+    });
+
+    logger.info(`Processed ${locations.length} locations`);
+  }
+
+  /**
+   * Determine the location on GitLab to be ingested base on configured groups and filters.
+   *
+   * @returns A list of location to be ingested
+   */
+  private async GetLocations() {
+    let res: Result = {
+      scanned: 0,
+      matches: [],
+    };
+
+    const groupToProcess = new Map<string, GitLabGroup>();
+
+    if (
+      this.config.groupPatterns !== undefined &&
+      this.config.groupPatterns.length > 0
+    ) {
+      for (const pattern of this.config.groupPatterns) {
+        const groups = paginated<GitLabGroup>(
+          options => this.gitLabClient.listGroups(options),
+          {
+            page: 1,
+            per_page: 50,
+          },
+        );
+
+        for await (const group of groups) {
+          if (
+            pattern.test(group.full_path) &&
+            !groupToProcess.has(group.full_path)
+          ) {
+            groupToProcess.set(group.full_path, group);
+          }
+        }
+      }
+
+      for (const group of groupToProcess.values()) {
+        const tmpRes = await this.GetProjectsToProcess(group.full_path);
+        res.scanned += tmpRes.scanned;
+        res.matches.push(...tmpRes.matches);
+      }
+    } else {
+      res = await this.GetProjectsToProcess(this.config.group);
+    }
+
+    const locations = this.deduplicateProjects(res.matches).map(p =>
+      this.createLocationSpec(p),
+    );
+
+    this.logger.info(
+      `Processed ${locations.length} from scanned ${res.scanned} projects.`,
+    );
+
+    return locations;
+  }
+
+  /**
+   * Deduplicate a list of projects based on their id.
+   *
+   * @param projects a list of projects to be deduplicated
+   * @returns a list of projects with unique id
+   */
+  private deduplicateProjects(projects: GitLabProject[]): GitLabProject[] {
+    const uniqueProjects = new Map<number, GitLabProject>();
+
+    for (const project of projects) {
+      uniqueProjects.set(project.id, project);
+    }
+
+    return Array.from(uniqueProjects.values());
+  }
+
+  /**
+   * Retrieve a list of projects that match configuration.
+   *
+   * @param group a full path of a GitLab group, can be empty
+   * @returns An array of project to be processed and the number of project scanned
+   */
+  private async GetProjectsToProcess(group: string) {
+    const res: Result = {
+      scanned: 0,
+      matches: [],
+    };
+
     const projects = paginated<GitLabProject>(
       options => this.gitLabClient.listProjects(options),
       {
-        group: this.config.group,
+        group: group,
         page: 1,
         per_page: 50,
         ...(!this.config.includeArchivedRepos && { archived: false }),
@@ -219,30 +317,15 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       },
     );
 
-    const res: Result = {
-      scanned: 0,
-      matches: [],
-    };
-
     for await (const project of projects) {
+      res.scanned++;
+
       if (await this.shouldProcessProject(project, this.gitLabClient)) {
-        res.scanned++;
         res.matches.push(project);
       }
     }
 
-    const locations = res.matches.map(p => this.createLocationSpec(p));
-
-    logger.info(
-      `Processed ${locations.length} from scanned ${res.scanned} projects.`,
-    );
-    await this.connection.applyMutation({
-      type: 'full',
-      entities: locations.map(location => ({
-        locationKey: this.getProviderName(),
-        entity: locationSpecToLocationEntity({ location }),
-      })),
-    });
+    return res;
   }
 
   private createLocationSpec(project: GitLabProject): LocationSpec {

plugins/catalog-backend-module-gitlab/src/providers/config.test.ts

@@ -55,6 +55,7 @@ describe('config', () => {
         catalogFile: 'catalog-info.yaml',
         projectPattern: /[\s\S]*/,
         groupPattern: /[\s\S]*/,
+        groupPatterns: [],
         userPattern: /[\s\S]*/,
         orgEnabled: false,
         allowInherited: false,
@@ -101,6 +102,7 @@ describe('config', () => {
         catalogFile: 'custom-file.yaml',
         projectPattern: /[\s\S]*/,
         groupPattern: /[\s\S]*/,
+        groupPatterns: [],
         userPattern: /[\s\S]*/,
         orgEnabled: false,
         allowInherited: false,
@@ -147,6 +149,7 @@ describe('config', () => {
         catalogFile: 'custom-file.yaml',
         projectPattern: /[\s\S]*/,
         groupPattern: /[\s\S]*/,
+        groupPatterns: [],
         userPattern: /[\s\S]*/,
         orgEnabled: false,
         allowInherited: false,
@@ -193,6 +196,7 @@ describe('config', () => {
         catalogFile: 'custom-file.yaml',
         projectPattern: /[\s\S]*/,
         groupPattern: /[\s\S]*/,
+        groupPatterns: [],
         userPattern: /[\s\S]*/,
         orgEnabled: false,
         allowInherited: false,
@@ -240,6 +244,7 @@ describe('config', () => {
         catalogFile: 'custom-file.yaml',
         projectPattern: /[\s\S]*/,
         groupPattern: /[\s\S]*/,
+        groupPatterns: [],
         userPattern: /[\s\S]*/,
         orgEnabled: false,
         allowInherited: false,
@@ -288,6 +293,7 @@ describe('config', () => {
         catalogFile: 'catalog-info.yaml',
         projectPattern: /[\s\S]*/,
         groupPattern: /[\s\S]*/,
+        groupPatterns: [],
         userPattern: /[\s\S]*/,
         orgEnabled: false,
         allowInherited: false,

plugins/catalog-backend-module-gitlab/src/providers/config.ts

@@ -42,6 +42,10 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
   const groupPattern = new RegExp(
     config.getOptionalString('groupPattern') ?? /[\s\S]*/,
   );
+  const groupPatterns: Array<RegExp> =
+    config
+      .getOptionalStringArray('groupPatterns')
+      ?.map(pattern => new RegExp(pattern)) ?? [];
   const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false;
   const allowInherited: boolean =
     config.getOptionalBoolean('allowInherited') ?? false;
@@ -81,6 +85,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
     projectPattern,
     userPattern,
     groupPattern,
+    groupPatterns,
     schedule,
     orgEnabled,
     allowInherited,