diff --git a/.changeset/chubby-results-relax.md b/.changeset/chubby-results-relax.md new file mode 100644 index 0000000000..c20c79085f --- /dev/null +++ b/.changeset/chubby-results-relax.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Added support for multiple group patterns instead of a single one to increase flexibility when filtering groups from GitLab. diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index e0ededd1fa..0fce7b864e 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -151,8 +151,11 @@ catalog: skipForkedRepos: false # Optional. If the project is a fork, skip repository includeArchivedRepos: false # Optional. If project is archived, include repository group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned + groupPattern: # Optional. Filters for groups based on a list of RegEx. Default, no filters. + - '^somegroup$' + - 'anothergroup' entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` - projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything + projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything excludeRepos: [] # Optional. A list of project paths that should be excluded from discovery, e.g. group/subgroup/repo. Should not start or end with a slash. schedule: # Same options as in SchedulerServiceTaskScheduleDefinition. Optional for the Legacy Backend System # supports cron, ISO duration, "human duration" as used in code diff --git a/plugins/catalog-backend-module-gitlab/config.d.ts b/plugins/catalog-backend-module-gitlab/config.d.ts index 4cc7ebe844..8a2b6d8fa5 100644 --- a/plugins/catalog-backend-module-gitlab/config.d.ts +++ b/plugins/catalog-backend-module-gitlab/config.d.ts @@ -72,7 +72,7 @@ export interface Config { /** * (Optional) RegExp for the Group Name Pattern */ - groupPattern?: string; + groupPattern?: string | string[]; /** * Specifies the types of group membership relations that should be included when ingesting data. * diff --git a/plugins/catalog-backend-module-gitlab/report.api.md b/plugins/catalog-backend-module-gitlab/report.api.md index 5b7bfadf1d..f5048835c1 100644 --- a/plugins/catalog-backend-module-gitlab/report.api.md +++ b/plugins/catalog-backend-module-gitlab/report.api.md @@ -111,7 +111,7 @@ export type GitlabProviderConfig = { catalogFile: string; projectPattern: RegExp; userPattern: RegExp; - groupPattern: RegExp; + groupPattern: RegExp | RegExp[]; allowInherited?: boolean; relations?: string[]; orgEnabled?: boolean; diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts index 0732bbc4cd..9af00d2564 100644 --- a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts @@ -81,6 +81,13 @@ const httpHandlers = [ ); }), + rest.get(`${apiBaseUrl}/groups/group1`, (_, res, ctx) => { + return res( + ctx.set('x-next-page', ''), + ctx.json(all_groups_response.find(g => g.full_path === 'group1')), + ); + }), + rest.get(`${apiBaseUrl}/groups/42`, (_, res, ctx) => { return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); }), @@ -229,6 +236,26 @@ const httpGroupListDescendantProjectsByName = all_groups_response.map(group => { }, ); }); + +const httpGroupListDescendantProjectsByFullPath = all_groups_response.map( + group => { + return rest.get( + `${apiBaseUrl}/groups/${encodeURIComponent(group.full_path)}/projects`, + (req, res, ctx) => { + const archived = req.url.searchParams.get('archived'); + + const projectsInGroup = all_projects_response.filter( + p => + p.path_with_namespace?.includes(group.full_path) && + (archived === 'false' ? !p.archived : true), + ); + + return res(ctx.json(projectsInGroup)); + }, + ); + }, +); + const httpGroupFindByNameDynamic = all_groups_response.map(group => { return rest.get(`${apiBaseUrl}/groups/${group.name}`, (_, res, ctx) => { return res(ctx.json(all_groups_response.find(g => g.name === group.name))); @@ -711,6 +738,7 @@ export const handlers = [ ...httpGroupFindByNameDynamic, ...httpGroupListDescendantProjectsById, ...httpGroupListDescendantProjectsByName, + ...httpGroupListDescendantProjectsByFullPath, ...graphqlHandlers, ...httpGroupFindByEncodedPathDynamic, ]; diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts index 2e74be66cc..d64b266183 100644 --- a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts @@ -820,6 +820,95 @@ export const config_org_group_with_subgroups_sass = { }, }, }; + +export const config_groupPatterns_only_noMatch = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: ['^group8$', '^group9$', '^group10$'], + }, + }, + }, + }, +}; + +export const config_groupPatterns_only_Match1Group = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: ['^group1$', '^group9$', '^group10$'], + }, + }, + }, + }, +}; + +export const config_groupPatterns_multiple_matches = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: ['^group1$', '^group2$', '^group10$'], + }, + }, + }, + }, +}; + +export const config_groupPatterns_duplicate_match = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: ['^group1$', 'subgroup1'], // Both patterns match the same group + }, + }, + }, + }, +}; + /** * GitLab API responses */ @@ -919,6 +1008,18 @@ export const all_projects_response: GitLabProject[] = [ web_url: 'https://example.com/group1/test-repo8-archived', path_with_namespace: 'group1/test-repo8-archived', }, + // project in subgroup + { + id: 9, + description: 'Project Nine Description', + name: 'test-repo9', + default_branch: 'main', + path: 'test-repo9', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/subgroup1/test-repo9', + path_with_namespace: 'group1/subgroup1/test-repo9', + }, ]; export const all_users_response: GitLabUser[] = [ @@ -1211,6 +1312,12 @@ export const all_groups_response: GitLabGroup[] = [ description: '', full_path: 'group1/subgroup2', }, + { + id: 8, + name: 'awsome-group', + description: '', + full_path: 'awsome-group', + }, ]; export const group_with_subgroups_response: GitLabGroup[] = [ diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 23b03ec366..aa2f73b35d 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -184,11 +184,13 @@ export type GitlabProviderConfig = { * Defaults to `[\s\S]*`, which means to not filter anything */ userPattern: RegExp; + /** - * Filters found groups based on provided patter. + * Filters found groups based on provided patterns. * Defaults to `[\s\S]*`, which means to not filter anything + * */ - groupPattern: RegExp; + groupPattern: RegExp | RegExp[]; /** * If true, the provider will also add inherited (ascendant) users to the ingested groups. diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index dd7a4f0d6d..f052434035 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -557,5 +557,121 @@ describe('GitlabDiscoveryEntityProvider - events', () => { }); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); }); - // EventSupportChange: stop add tests >>> + + it('should ignore projects when none of the groups regex patterns match', async () => { + const config = new ConfigReader(mock.config_groupPatterns_only_noMatch); + + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [], + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // No entities should be applied + }); + + it('should include only the project that matches one of the group regex patterns', async () => { + const config = new ConfigReader(mock.config_groupPatterns_only_Match1Group); + + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities_default_branch.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('/group1/'), // Only projects in 'group2' should match + ), + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // Only one matching project should be applied + }); + + it('should include projects that match multiple group regex patterns', async () => { + const config = new ConfigReader(mock.config_groupPatterns_multiple_matches); + + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities_default_branch.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('/group1/') || + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('/group2/'), + ), + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); // Projects from both groups should be applied + }); + + it('should not create duplicate locations when multiple groupPatterns match the same group', async () => { + const config = new ConfigReader(mock.config_groupPatterns_duplicate_match); + + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities_default_branch.filter(entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('/group1/'), + ), + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index 8bf749c275..08bd498615 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -33,6 +33,7 @@ import { WebhookProjectSchema, WebhookPushEventSchema } from '@gitbeaker/rest'; import * as uuid from 'uuid'; import { GitLabClient, + GitLabGroup, GitLabProject, GitlabProviderConfig, paginated, @@ -207,10 +208,119 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { ); } + const locations = await this.getEntities(); + + await this.connection.applyMutation({ + type: 'full', + entities: locations.map(location => ({ + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + })), + }); + + logger.info(`Processed ${locations.length} locations`); + } + + /** + * Determine the location on GitLab to be ingested base on configured groups and filters. + * + * @returns A list of location to be ingested + */ + private async getEntities() { + let res: Result = { + scanned: 0, + matches: [], + }; + + const groupToProcess = new Map(); + let groupFilters; + + if (this.config.groupPattern !== undefined) { + const patterns = Array.isArray(this.config.groupPattern) + ? this.config.groupPattern + : [this.config.groupPattern]; + + if (patterns.length === 1 && patterns[0].source === '[\\s\\S]*') { + // if the pattern is a catch-all, we don't need to filter groups + groupFilters = new Array(); + } else { + groupFilters = patterns; + } + } + + if (groupFilters && groupFilters.length > 0) { + const groups = paginated( + options => this.gitLabClient.listGroups(options), + { + page: 1, + per_page: 50, + }, + ); + + for await (const group of groups) { + if ( + groupFilters.some(groupFilter => groupFilter.test(group.full_path)) && + !groupToProcess.has(group.full_path) + ) { + groupToProcess.set(group.full_path, group); + } + } + + for (const group of groupToProcess.values()) { + const tmpRes = await this.getProjectsToProcess(group.full_path); + res.scanned += tmpRes.scanned; + // merge both arrays safely + for (const project of tmpRes.matches) { + res.matches.push(project); + } + } + } else { + res = await this.getProjectsToProcess(this.config.group); + } + + const locations = this.deduplicateProjects(res.matches).map(p => + this.createLocationSpec(p), + ); + + this.logger.info( + `Processed ${locations.length} from scanned ${res.scanned} projects.`, + ); + + return locations; + } + + /** + * Deduplicate a list of projects based on their id. + * + * @param projects - a list of projects to be deduplicated + * @returns a list of projects with unique id + */ + private deduplicateProjects(projects: GitLabProject[]): GitLabProject[] { + const uniqueProjects = new Map(); + + for (const project of projects) { + uniqueProjects.set(project.id, project); + } + + return Array.from(uniqueProjects.values()); + } + + /** + * Retrieve a list of projects that match configuration. + * + * @param group - a full path of a GitLab group, can be empty + * @returns An array of project to be processed and the number of project scanned + */ + private async getProjectsToProcess(group: string) { + const res: Result = { + scanned: 0, + matches: [], + }; + const projects = paginated( options => this.gitLabClient.listProjects(options), { - group: this.config.group, + group: group, page: 1, per_page: 50, ...(!this.config.includeArchivedRepos && { archived: false }), @@ -219,30 +329,15 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }, ); - const res: Result = { - scanned: 0, - matches: [], - }; - for await (const project of projects) { + res.scanned++; + if (await this.shouldProcessProject(project, this.gitLabClient)) { - res.scanned++; res.matches.push(project); } } - const locations = res.matches.map(p => this.createLocationSpec(p)); - - logger.info( - `Processed ${locations.length} from scanned ${res.scanned} projects.`, - ); - await this.connection.applyMutation({ - type: 'full', - entities: locations.map(location => ({ - locationKey: this.getProviderName(), - entity: locationSpecToLocationEntity({ location }), - })), - }); + return res; } private createLocationSpec(project: GitLabProject): LocationSpec { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 3eec377e73..3b8720b5f2 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -111,6 +111,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private groupEntitiesTransformer: GroupEntitiesTransformer; private groupNameTransformer: GroupNameTransformer; private readonly gitLabClient: GitLabClient; + private readonly groupPatterns: RegExp[]; static fromConfig( config: Config, @@ -198,6 +199,10 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { this.groupNameTransformer = options.groupNameTransformer ?? defaultGroupNameTransformer; + this.groupPatterns = Array.isArray(this.config.groupPattern) + ? this.config.groupPattern + : [this.config.groupPattern]; + this.gitLabClient = new GitLabClient({ config: this.integration.config, logger: this.logger, @@ -464,7 +469,6 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { for await (const group of groups) { groupRes.scanned++; - if (!this.shouldProcessGroup(group)) { logger.debug(`Skipped group: ${group.full_path}`); continue; @@ -798,7 +802,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private shouldProcessGroup(group: GitLabGroup): boolean { return ( - this.config.groupPattern.test(group.full_path) && + this.groupPatterns.some(pattern => pattern.test(group.full_path)) && (!this.config.group || group.full_path.startsWith(`${this.config.group}/`) || group.full_path === this.config.group) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index 5b50fce5b1..f635d4c432 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -39,9 +39,21 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { const userPattern = new RegExp( config.getOptionalString('userPattern') ?? /[\s\S]*/, ); - const groupPattern = new RegExp( - config.getOptionalString('groupPattern') ?? /[\s\S]*/, - ); + + const configValue = config.getOptional('groupPattern'); + let groupPattern; + + if ((configValue && typeof configValue === 'string') || !configValue) { + groupPattern = new RegExp( + config.getOptionalString('groupPattern') ?? /[\s\S]*/, + ); + } else if (configValue && Array.isArray(configValue)) { + const configPattern = config.getOptionalStringArray('groupPattern') ?? []; + groupPattern = configPattern.map(pattern => new RegExp(pattern)); + } else { + groupPattern = new RegExp(/[\s\S]*/); + } + const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false; const allowInherited: boolean = config.getOptionalBoolean('allowInherited') ?? false;