From ecf12e5de30dcc522e993f6755de7dc3af4d7cf9 Mon Sep 17 00:00:00 2001 From: alessandro Date: Fri, 7 Jul 2023 06:42:23 +0200 Subject: [PATCH 1/5] feat(catalog-backend-module-gitlab): Added option to skip forked repos in GitlabDiscoveryEntityProvider config Signed-off-by: alessandro --- docs/integrations/gitlab/discovery.md | 1 + plugins/catalog-backend-module-gitlab/src/lib/types.ts | 1 + .../src/providers/GitlabDiscoveryEntityProvider.ts | 7 +++++++ .../catalog-backend-module-gitlab/src/providers/config.ts | 3 +++ 4 files changed, 12 insertions(+) diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 4ddb0413b0..235d93ab03 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -23,6 +23,7 @@ catalog: host: gitlab-host # Identifies one of the hosts set up in the integrations branch: main # Optional. Used to discover on a specific branch fallbackBranch: main # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default + skipForkedRepos: false # Optional. If the project is a fork, skip repository group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 341e2a9d9a..9b3a68c107 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -90,4 +90,5 @@ export type GitlabProviderConfig = { groupPattern: RegExp; orgEnabled?: boolean; schedule?: TaskScheduleDefinition; + skipForkedRepos?: boolean; }; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index 6001a32a4a..08d2fe48a7 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -177,6 +177,13 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { continue; } + if ( + this.config.skipForkedRepos && + project.hasOwnProperty('forked_from_project') + ) { + continue; + } + if ( !this.config.branch && this.config.fallbackBranch === '*' && diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index cd15c4825c..c34cb81808 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -43,6 +43,8 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { config.getOptionalString('groupPattern') ?? /[\s\S]*/, ); const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false; + const skipForkedRepos: boolean = + config.getOptionalBoolean('skipForkedRepos') ?? false; const schedule = config.has('schedule') ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) @@ -60,6 +62,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { groupPattern, schedule, orgEnabled, + skipForkedRepos, }; } From e6c721439f37aeb723d121aed96dbde4f86284d9 Mon Sep 17 00:00:00 2001 From: alessandro Date: Fri, 7 Jul 2023 06:44:11 +0200 Subject: [PATCH 2/5] feat(catalog-backend-module-gitlab): Added changeset Signed-off-by: alessandro --- .changeset/khaki-flies-draw.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/khaki-flies-draw.md diff --git a/.changeset/khaki-flies-draw.md b/.changeset/khaki-flies-draw.md new file mode 100644 index 0000000000..ada45dea06 --- /dev/null +++ b/.changeset/khaki-flies-draw.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Added option to skip forked repos in GitlabDiscoveryEntityProvider From d913b37a8b32c7fcf72a79982c4e3157cda9807d Mon Sep 17 00:00:00 2001 From: alessandro Date: Fri, 7 Jul 2023 06:50:55 +0200 Subject: [PATCH 3/5] feat(catalog-backend-module-gitlab): Added tests Signed-off-by: alessandro --- .../GitlabDiscoveryEntityProvider.test.ts | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index 75a4c6414c..7508b35b84 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -343,6 +343,119 @@ describe('GitlabDiscoveryEntityProvider', () => { }); }); + it('should filter fork projects', async () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + skipForkedRepos: true, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + server.use( + rest.get( + `https://api.gitlab.example/api/v4/projects`, + (_req, res, ctx) => { + const response = [ + { + id: 123, + default_branch: 'master', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://api.gitlab.example/test-group/test-repo', + path_with_namespace: 'test-group/test-repo', + forked_from_project: { + id: 13083, + }, + }, + { + id: 124, + default_branch: 'master', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://api.gitlab.example/john/example', + path_with_namespace: 'john/example', + }, + ]; + return res(ctx.json(response)); + }, + ), + rest.head( + 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', + (req, res, ctx) => { + if (req.url.searchParams.get('ref') === 'master') { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ), + rest.head( + 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', + (req, res, ctx) => { + if (req.url.searchParams.get('ref') === 'master') { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ), + ); + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', + 'backstage.io/managed-by-origin-location': + 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', + }, + name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9', + }, + spec: { + presence: 'optional', + target: + 'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }, + ], + }); + }); + it('fail without schedule and scheduler', () => { const config = new ConfigReader({ integrations: { From 0de20357eee659a8bfa7fef6fe50d8ba851da90f Mon Sep 17 00:00:00 2001 From: alessandro Date: Fri, 7 Jul 2023 07:05:33 +0200 Subject: [PATCH 4/5] feat(catalog-backend-module-gitlab): Fix tests Signed-off-by: alessandro --- .../src/providers/config.test.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index 1035538939..bac56cc6f3 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -59,6 +59,7 @@ describe('config', () => { userPattern: /[\s\S]*/, orgEnabled: false, schedule: undefined, + skipForkedRepos: false, }), ); }); @@ -95,6 +96,45 @@ describe('config', () => { userPattern: /[\s\S]*/, orgEnabled: false, schedule: undefined, + skipForkedRepos: false, + }), + ); + }); + + it('valid config with skipForkedRepos', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + group: 'group', + host: 'host', + branch: 'not-master', + fallbackBranch: 'main', + entityFilename: 'custom-file.yaml', + skipForkedRepos: true, + }, + }, + }, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(1); + result.forEach(r => + expect(r).toStrictEqual({ + id: 'test', + group: 'group', + branch: 'not-master', + fallbackBranch: 'main', + host: 'host', + catalogFile: 'custom-file.yaml', + projectPattern: /[\s\S]*/, + groupPattern: /[\s\S]*/, + userPattern: /[\s\S]*/, + orgEnabled: false, + schedule: undefined, + skipForkedRepos: true, }), ); }); @@ -133,6 +173,7 @@ describe('config', () => { groupPattern: /[\s\S]*/, userPattern: /[\s\S]*/, orgEnabled: false, + skipForkedRepos: false, schedule: { frequency: Duration.fromISO('PT30M'), timeout: { From 941aa42ef65fc0e435167db837439121e4cbeb7e Mon Sep 17 00:00:00 2001 From: alessandro Date: Thu, 27 Jul 2023 12:40:49 +0200 Subject: [PATCH 5/5] feat(catalog-backend-module-gitlab): Add skipForkedRepos field in config.d.ts Signed-off-by: alessandro --- plugins/catalog-backend-module-gitlab/config.d.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/config.d.ts b/plugins/catalog-backend-module-gitlab/config.d.ts index 00ad0ef032..f5afc71093 100644 --- a/plugins/catalog-backend-module-gitlab/config.d.ts +++ b/plugins/catalog-backend-module-gitlab/config.d.ts @@ -60,6 +60,10 @@ export interface Config { * (Optional) RegExp for the Group Name Pattern */ groupPattern?: RegExp; + /** + * (Optional) Skip forked repository + */ + skipForkedRepos?: boolean; } >; };