diff --git a/.changeset/chubby-results-relax.md b/.changeset/chubby-results-relax.md new file mode 100644 index 0000000000..a668570694 --- /dev/null +++ b/.changeset/chubby-results-relax.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': minor +--- + +Added support for multiple group patterns instead of a single one to increase flexibility when filtering groups from GitLab. diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index e0ededd1fa..85daea3649 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -151,6 +151,9 @@ catalog: skipForkedRepos: false # Optional. If the project is a fork, skip repository includeArchivedRepos: false # Optional. If project is archived, include repository group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned + groupPatterns: # Optional. Filters for groups based on a list of patterns. Default, no filters. + - '^somegroup$' + - 'anothergroup' entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything excludeRepos: [] # Optional. A list of project paths that should be excluded from discovery, e.g. group/subgroup/repo. Should not start or end with a slash. diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts index 0732bbc4cd..7549e6faca 100644 --- a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts @@ -229,6 +229,26 @@ const httpGroupListDescendantProjectsByName = all_groups_response.map(group => { }, ); }); + +const httpGroupListDescendantProjectsByFullPath = all_groups_response.map( + group => { + return rest.get( + `${apiBaseUrl}/groups/${encodeURIComponent(group.full_path)}/projects`, + (req, res, ctx) => { + const archived = req.url.searchParams.get('archived'); + + const projectsInGroup = all_projects_response.filter( + p => + p.path_with_namespace?.includes(group.full_path) && + (archived === 'false' ? !p.archived : true), + ); + + return res(ctx.json(projectsInGroup)); + }, + ); + }, +); + const httpGroupFindByNameDynamic = all_groups_response.map(group => { return rest.get(`${apiBaseUrl}/groups/${group.name}`, (_, res, ctx) => { return res(ctx.json(all_groups_response.find(g => g.name === group.name))); @@ -711,6 +731,7 @@ export const handlers = [ ...httpGroupFindByNameDynamic, ...httpGroupListDescendantProjectsById, ...httpGroupListDescendantProjectsByName, + ...httpGroupListDescendantProjectsByFullPath, ...graphqlHandlers, ...httpGroupFindByEncodedPathDynamic, ]; diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts index 2e74be66cc..768ccb416b 100644 --- a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts @@ -820,6 +820,95 @@ export const config_org_group_with_subgroups_sass = { }, }, }; + +export const config_groupPatterns_only_noMatch = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPatterns: ['^group8$', '^group9$', '^group10$'], + }, + }, + }, + }, +}; + +export const config_groupPatterns_only_Match1Group = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPatterns: ['^group1$', '^group9$', '^group10$'], + }, + }, + }, + }, +}; + +export const config_groupPatterns_multiple_matches = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPatterns: ['^group1$', '^group2$', '^group10$'], + }, + }, + }, + }, +}; + +export const config_groupPatterns_duplicate_match = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPatterns: ['^group1$', 'subgroup1'], // Both patterns match the same group + }, + }, + }, + }, +}; + /** * GitLab API responses */ @@ -919,6 +1008,18 @@ export const all_projects_response: GitLabProject[] = [ web_url: 'https://example.com/group1/test-repo8-archived', path_with_namespace: 'group1/test-repo8-archived', }, + // project in subgroup + { + id: 9, + description: 'Project Nine Description', + name: 'test-repo9', + default_branch: 'main', + path: 'test-repo9', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/subgroup1/test-repo9', + path_with_namespace: 'group1/subgroup1/test-repo9', + }, ]; export const all_users_response: GitLabUser[] = [ diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 23b03ec366..a51e129fbf 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -184,12 +184,20 @@ export type GitlabProviderConfig = { * Defaults to `[\s\S]*`, which means to not filter anything */ userPattern: RegExp; + /** - * Filters found groups based on provided patter. + * Filters found groups based on provided pattern. * Defaults to `[\s\S]*`, which means to not filter anything + * + * @deprecated Use groupPatterns that provide a list instead. */ groupPattern: RegExp; + /** + * Optional, provide a list of pattern for each can match a list of groups. + */ + groupPatterns: RegExp[]; + /** * If true, the provider will also add inherited (ascendant) users to the ingested groups. * See: https://docs.gitlab.com/ee/api/graphql/reference/#groupmemberrelation diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index dd7a4f0d6d..7d7bb699f5 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -558,4 +558,125 @@ describe('GitlabDiscoveryEntityProvider - events', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); }); // EventSupportChange: stop add tests >>> + + // add the following test: + // setup: 3 projects, 2 groups + // 1 test where none match --> OK + // 1 test where match only 1 group --> OK + // 1 test where match 2 groups + // 1 test where match 1 group with 2 identical regex + + it('should ignore projects when none of the groups regex patterns match', async () => { + const config = new ConfigReader(mock.config_groupPatterns_only_noMatch); + + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [], + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // No entities should be applied + }); +}); +it('should include only the project that matches one of the group regex patterns', async () => { + const config = new ConfigReader(mock.config_groupPatterns_only_Match1Group); + + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities_default_branch.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('/group1/'), // Only projects in 'group2' should match + ), + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // Only one matching project should be applied +}); +it('should include projects that match multiple group regex patterns', async () => { + const config = new ConfigReader(mock.config_groupPatterns_multiple_matches); + + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities_default_branch.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('/group1/') || + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('/group2/'), + ), + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // Projects from both groups should be applied +}); +it('should not create duplicate locations when multiple groupPatterns match the same group', async () => { + const config = new ConfigReader(mock.config_groupPatterns_duplicate_match); + + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities_default_branch.filter(entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('/group1/'), + ), + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index 8bf749c275..3a7f1ab82d 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -33,6 +33,7 @@ import { WebhookProjectSchema, WebhookPushEventSchema } from '@gitbeaker/rest'; import * as uuid from 'uuid'; import { GitLabClient, + GitLabGroup, GitLabProject, GitlabProviderConfig, paginated, @@ -207,10 +208,107 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { ); } + const locations = await this.GetLocations(); + + await this.connection.applyMutation({ + type: 'full', + entities: locations.map(location => ({ + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + })), + }); + + logger.info(`Processed ${locations.length} locations`); + } + + /** + * Determine the location on GitLab to be ingested base on configured groups and filters. + * + * @returns A list of location to be ingested + */ + private async GetLocations() { + let res: Result = { + scanned: 0, + matches: [], + }; + + const groupToProcess = new Map(); + + if ( + this.config.groupPatterns !== undefined && + this.config.groupPatterns.length > 0 + ) { + for (const pattern of this.config.groupPatterns) { + const groups = paginated( + options => this.gitLabClient.listGroups(options), + { + page: 1, + per_page: 50, + }, + ); + + for await (const group of groups) { + if ( + pattern.test(group.full_path) && + !groupToProcess.has(group.full_path) + ) { + groupToProcess.set(group.full_path, group); + } + } + } + + for (const group of groupToProcess.values()) { + const tmpRes = await this.GetProjectsToProcess(group.full_path); + res.scanned += tmpRes.scanned; + res.matches.push(...tmpRes.matches); + } + } else { + res = await this.GetProjectsToProcess(this.config.group); + } + + const locations = this.deduplicateProjects(res.matches).map(p => + this.createLocationSpec(p), + ); + + this.logger.info( + `Processed ${locations.length} from scanned ${res.scanned} projects.`, + ); + + return locations; + } + + /** + * Deduplicate a list of projects based on their id. + * + * @param projects a list of projects to be deduplicated + * @returns a list of projects with unique id + */ + private deduplicateProjects(projects: GitLabProject[]): GitLabProject[] { + const uniqueProjects = new Map(); + + for (const project of projects) { + uniqueProjects.set(project.id, project); + } + + return Array.from(uniqueProjects.values()); + } + + /** + * Retrieve a list of projects that match configuration. + * + * @param group a full path of a GitLab group, can be empty + * @returns An array of project to be processed and the number of project scanned + */ + private async GetProjectsToProcess(group: string) { + const res: Result = { + scanned: 0, + matches: [], + }; + const projects = paginated( options => this.gitLabClient.listProjects(options), { - group: this.config.group, + group: group, page: 1, per_page: 50, ...(!this.config.includeArchivedRepos && { archived: false }), @@ -219,30 +317,15 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }, ); - const res: Result = { - scanned: 0, - matches: [], - }; - for await (const project of projects) { + res.scanned++; + if (await this.shouldProcessProject(project, this.gitLabClient)) { - res.scanned++; res.matches.push(project); } } - const locations = res.matches.map(p => this.createLocationSpec(p)); - - logger.info( - `Processed ${locations.length} from scanned ${res.scanned} projects.`, - ); - await this.connection.applyMutation({ - type: 'full', - entities: locations.map(location => ({ - locationKey: this.getProviderName(), - entity: locationSpecToLocationEntity({ location }), - })), - }); + return res; } private createLocationSpec(project: GitLabProject): LocationSpec { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index 8790a3fe53..27dd32367a 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -55,6 +55,7 @@ describe('config', () => { catalogFile: 'catalog-info.yaml', projectPattern: /[\s\S]*/, groupPattern: /[\s\S]*/, + groupPatterns: [], userPattern: /[\s\S]*/, orgEnabled: false, allowInherited: false, @@ -101,6 +102,7 @@ describe('config', () => { catalogFile: 'custom-file.yaml', projectPattern: /[\s\S]*/, groupPattern: /[\s\S]*/, + groupPatterns: [], userPattern: /[\s\S]*/, orgEnabled: false, allowInherited: false, @@ -147,6 +149,7 @@ describe('config', () => { catalogFile: 'custom-file.yaml', projectPattern: /[\s\S]*/, groupPattern: /[\s\S]*/, + groupPatterns: [], userPattern: /[\s\S]*/, orgEnabled: false, allowInherited: false, @@ -193,6 +196,7 @@ describe('config', () => { catalogFile: 'custom-file.yaml', projectPattern: /[\s\S]*/, groupPattern: /[\s\S]*/, + groupPatterns: [], userPattern: /[\s\S]*/, orgEnabled: false, allowInherited: false, @@ -240,6 +244,7 @@ describe('config', () => { catalogFile: 'custom-file.yaml', projectPattern: /[\s\S]*/, groupPattern: /[\s\S]*/, + groupPatterns: [], userPattern: /[\s\S]*/, orgEnabled: false, allowInherited: false, @@ -288,6 +293,7 @@ describe('config', () => { catalogFile: 'catalog-info.yaml', projectPattern: /[\s\S]*/, groupPattern: /[\s\S]*/, + groupPatterns: [], userPattern: /[\s\S]*/, orgEnabled: false, allowInherited: false, diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index 5b50fce5b1..639a1e1177 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -42,6 +42,10 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { const groupPattern = new RegExp( config.getOptionalString('groupPattern') ?? /[\s\S]*/, ); + const groupPatterns: Array = + config + .getOptionalStringArray('groupPatterns') + ?.map(pattern => new RegExp(pattern)) ?? []; const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false; const allowInherited: boolean = config.getOptionalBoolean('allowInherited') ?? false; @@ -81,6 +85,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { projectPattern, userPattern, groupPattern, + groupPatterns, schedule, orgEnabled, allowInherited,