diff --git a/.changeset/red-radios-promise.md b/.changeset/red-radios-promise.md new file mode 100644 index 0000000000..ea876c3480 --- /dev/null +++ b/.changeset/red-radios-promise.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Adds new optional `excludeRepos` configuration option to the Gitlab catalog provider. diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 13ffd79801..70f12f181e 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -153,6 +153,7 @@ catalog: group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything + excludeRepos: [] # Optional. A list of project paths that should be excluded from discovery, e.g. group/subgroup/repo. Should not start or end with a slash. schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System # supports cron, ISO duration, "human duration" as used in code frequency: { minutes: 30 } diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index 2b78c67205..0a1e4b7d36 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -110,6 +110,7 @@ export type GitlabProviderConfig = { orgEnabled?: boolean; schedule?: TaskScheduleDefinition; skipForkedRepos?: boolean; + excludeRepos?: string[]; }; // @public diff --git a/plugins/catalog-backend-module-gitlab/config.d.ts b/plugins/catalog-backend-module-gitlab/config.d.ts index 7d183b6e58..9f009ee478 100644 --- a/plugins/catalog-backend-module-gitlab/config.d.ts +++ b/plugins/catalog-backend-module-gitlab/config.d.ts @@ -63,6 +63,11 @@ export interface Config { * (Optional) Skip forked repository */ skipForkedRepos?: boolean; + /** + * (Optional) A list of strings containing the paths of the repositories to skip + * Should be in the format group/subgroup/repo, with no leading or trailing slashes. + */ + excludeRepos?: string[]; }; }; }; diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts index 798df32695..4a173c6b50 100644 --- a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts @@ -299,6 +299,33 @@ export const config_single_integration_skip_forks: MockObject = { }, }; +export const config_single_integration_exclude_repos: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + excludeRepos: ['group1/test-repo1'], + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + export const config_no_schedule: MockObject = { integrations: { gitlab: [ diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 0b9ea6a868..69c8ed5f92 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -201,6 +201,11 @@ export type GitlabProviderConfig = { * If the project is a fork, skip repository */ skipForkedRepos?: boolean; + /** + * List of repositories to exclude from discovery, should be the full path to the repository, e.g. `group/project` + * Paths should not start or end with a slash. + */ + excludeRepos?: string[]; }; /** diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index 473e547a57..f5e167e564 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -130,6 +130,7 @@ describe('GitlabDiscoveryEntityProvider - configuration', () => { ); }); }); + describe('GitlabDiscoveryEntityProvider - refresh', () => { it('should apply full update on scheduled execution', async () => { const config = new ConfigReader(mock.config_no_org_integration); @@ -231,6 +232,38 @@ describe('GitlabDiscoveryEntityProvider - refresh', () => { }); }); + it('should filter repositories that are excluded', async () => { + const config = new ConfigReader( + mock.config_single_integration_exclude_repos, + ); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities_default_branch.filter( + entity => + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('test-repo1') && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), + }); + }); + // branch and fallback branch are undefined in the config it('should ingest catalog from project default branch only', async () => { const config = new ConfigReader(mock.config_single_integration); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index b2ef475d9d..bbd90795be 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -470,6 +470,13 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { return false; } + if (this.config.excludeRepos?.includes(project.path_with_namespace ?? '')) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it is excluded.`, + ); + return false; + } + const project_branch = this.config.branch ?? project.default_branch ?? diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index 4418dba50b..28df8c88b7 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -60,6 +60,7 @@ describe('config', () => { allowInherited: false, schedule: undefined, skipForkedRepos: false, + excludeRepos: [], restrictUsersToGroup: false, }), ); @@ -99,6 +100,7 @@ describe('config', () => { allowInherited: false, schedule: undefined, skipForkedRepos: false, + excludeRepos: [], restrictUsersToGroup: false, }), ); @@ -139,11 +141,54 @@ describe('config', () => { allowInherited: false, schedule: undefined, restrictUsersToGroup: false, + excludeRepos: [], skipForkedRepos: true, }), ); }); + it('valid config with excludeRepos', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + group: 'group', + host: 'host', + branch: 'not-master', + fallbackBranch: 'main', + entityFilename: 'custom-file.yaml', + skipForkedRepos: false, + excludeRepos: ['foo/bar', 'quz/qux'], + }, + }, + }, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(1); + result.forEach(r => + expect(r).toStrictEqual({ + id: 'test', + group: 'group', + branch: 'not-master', + fallbackBranch: 'main', + host: 'host', + catalogFile: 'custom-file.yaml', + projectPattern: /[\s\S]*/, + groupPattern: /[\s\S]*/, + userPattern: /[\s\S]*/, + orgEnabled: false, + allowInherited: false, + schedule: undefined, + restrictUsersToGroup: false, + skipForkedRepos: false, + excludeRepos: ['foo/bar', 'quz/qux'], + }), + ); + }); + it('valid config with schedule', () => { const config = new ConfigReader({ catalog: { @@ -181,6 +226,7 @@ describe('config', () => { allowInherited: false, skipForkedRepos: false, restrictUsersToGroup: false, + excludeRepos: [], schedule: { frequency: { minutes: 30 }, timeout: { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index 68eba7165a..d26cba5328 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -47,6 +47,8 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { config.getOptionalBoolean('allowInherited') ?? false; const skipForkedRepos: boolean = config.getOptionalBoolean('skipForkedRepos') ?? false; + const excludeRepos: string[] = + config.getOptionalStringArray('excludeRepos') ?? []; const schedule = config.has('schedule') ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) @@ -68,6 +70,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { orgEnabled, allowInherited, skipForkedRepos, + excludeRepos, restrictUsersToGroup, }; }