From be129f8f3cd15b4e10950450bb8ca3ec8a74daf2 Mon Sep 17 00:00:00 2001 From: Matteo Silvestri Date: Mon, 20 Feb 2023 16:24:51 +0100 Subject: [PATCH 1/8] filter gitlab groups by prefix Signed-off-by: Matteo Silvestri --- .changeset/wild-bulldogs-suffer.md | 5 ++++ .../GitlabOrgDiscoveryEntityProvider.ts | 28 ++++++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) create mode 100644 .changeset/wild-bulldogs-suffer.md diff --git a/.changeset/wild-bulldogs-suffer.md b/.changeset/wild-bulldogs-suffer.md new file mode 100644 index 0000000000..02f50927e9 --- /dev/null +++ b/.changeset/wild-bulldogs-suffer.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +filter gitlab groups by prefix diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index f90a61f9e9..c507b19549 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -199,6 +199,10 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { continue; } + if (!group.full_path.startsWith(this.config.group)) { + continue; + } + groupRes.scanned++; groupRes.matches.push(group); @@ -253,7 +257,11 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { type: 'full', entities: [...userEntities, ...groupEntities].map(entity => ({ locationKey: this.getProviderName(), - entity: this.withLocations(this.integration.config.host, entity), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), })), }); } @@ -273,7 +281,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const entity = this.createGroupEntity(group, host); if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) { - entity.spec.parent = idMapped[group.parent_id].full_path; + entity.spec.parent = this.groupName( + idMapped[group.parent_id].full_path, + ); } entities.push(entity); @@ -282,11 +292,11 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { return entities; } - private withLocations(host: string, entity: Entity): Entity { + private withLocations(host: string, baseUrl: string, entity: Entity): Entity { const location = entity.kind === 'Group' - ? `url:${host}/teams/${entity.metadata.name}` - : `url:${host}/${entity.metadata.name}`; + ? `url:${baseUrl}/${entity.metadata.annotations[`${host}/team-path`]}` + : `url:${baseUrl}/${entity.metadata.name}`; return merge( { metadata: { @@ -338,13 +348,17 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { if (!entity.spec.memberOf) { entity.spec.memberOf = []; } - entity.spec.memberOf.push(group.full_path.replace('/', '-')); + entity.spec.memberOf.push(this.groupName(group.full_path)); } } return entity; } + private groupName(full_path: string): string { + return full_path.replace(`${this.config.group}/`, '').replaceAll('/', '-'); + } + private createGroupEntity(group: GitLabGroup, host: string): GroupEntity { const annotations: { [annotationName: string]: string } = {}; @@ -354,7 +368,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { apiVersion: 'backstage.io/v1alpha1', kind: 'Group', metadata: { - name: group.full_path.replace('/', '-'), + name: this.groupName(group.full_path), annotations: annotations, }, spec: { From 4a1a720cd380d017b4dce8326d029e7161b1ddc2 Mon Sep 17 00:00:00 2001 From: Matteo Silvestri Date: Mon, 20 Feb 2023 16:37:48 +0100 Subject: [PATCH 2/8] fix GitlabOrgDiscoveryEntityProvider.test.ts tests Signed-off-by: Matteo Silvestri --- .../providers/GitlabOrgDiscoveryEntityProvider.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 74649f41da..268b4c3ba4 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -350,9 +350,10 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { kind: 'User', metadata: { annotations: { - 'backstage.io/managed-by-location': 'url:test-gitlab/test1', + 'backstage.io/managed-by-location': + 'url:https://test-gitlab/test1', 'backstage.io/managed-by-origin-location': - 'url:test-gitlab/test1', + 'url:https://test-gitlab/test1', 'test-gitlab/user-login': 'https://gitlab.example/test1', }, name: 'test1', @@ -375,9 +376,9 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:test-gitlab/teams/group1-group2', + 'url:https://test-gitlab/teams/group1-group2', 'backstage.io/managed-by-origin-location': - 'url:test-gitlab/teams/group1-group2', + 'url:https://test-gitlab/teams/group1-group2', 'test-gitlab/team-path': 'group1/group2', }, description: 'Group2', From fc2b7e885887e8497f57032eec2161b60cfe7bb8 Mon Sep 17 00:00:00 2001 From: Matteo Silvestri Date: Wed, 22 Feb 2023 10:35:37 +0100 Subject: [PATCH 3/8] fix GitlabOrgDiscoveryEntityProvider.test.ts Signed-off-by: Matteo Silvestri --- .../src/providers/GitlabOrgDiscoveryEntityProvider.test.ts | 5 ++--- .../src/providers/GitlabOrgDiscoveryEntityProvider.ts | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 268b4c3ba4..cff215acd3 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -184,7 +184,6 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { gitlab: { 'test-id': { host: 'test-gitlab', - group: 'test-group', orgEnabled: true, }, }, @@ -376,9 +375,9 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://test-gitlab/teams/group1-group2', + 'url:https://test-gitlab/group1/group2', 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/teams/group1-group2', + 'url:https://test-gitlab/group1/group2', 'test-gitlab/team-path': 'group1/group2', }, description: 'Group2', diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index c507b19549..77314ead17 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -356,7 +356,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { } private groupName(full_path: string): string { - return full_path.replace(`${this.config.group}/`, '').replaceAll('/', '-'); + if (this.config.group && full_path.startsWith(`${this.config.group}/`)) { + return full_path + .replace(`${this.config.group}/`, '') + .replaceAll('/', '-'); + } + return full_path.replaceAll('/', '-'); } private createGroupEntity(group: GitLabGroup, host: string): GroupEntity { From 3d96343887195b3838508492a1e967a769e497f5 Mon Sep 17 00:00:00 2001 From: Matteo Silvestri Date: Wed, 22 Feb 2023 13:08:12 +0100 Subject: [PATCH 4/8] fix compilation issue GitlabOrgDiscoveryEntityProvider Signed-off-by: Matteo Silvestri --- .../src/providers/GitlabOrgDiscoveryEntityProvider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 77314ead17..57a7210e24 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -295,7 +295,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private withLocations(host: string, baseUrl: string, entity: Entity): Entity { const location = entity.kind === 'Group' - ? `url:${baseUrl}/${entity.metadata.annotations[`${host}/team-path`]}` + ? `url:${baseUrl}/${entity.metadata.annotations?.[`${host}/team-path`]}` : `url:${baseUrl}/${entity.metadata.name}`; return merge( { From 2de756939b7f0c36734e9f90e98df52aff26a130 Mon Sep 17 00:00:00 2001 From: Matteo Silvestri Date: Wed, 22 Feb 2023 15:26:29 +0100 Subject: [PATCH 5/8] fix GitlabOrgDiscoveryEntityProvider and tests Signed-off-by: Matteo Silvestri --- .../src/providers/GitlabOrgDiscoveryEntityProvider.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 57a7210e24..43c070ecae 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -199,7 +199,10 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { continue; } - if (!group.full_path.startsWith(this.config.group)) { + if ( + this.config.group && + !group.full_path.startsWith(`${this.config.group}/`) + ) { continue; } From c47a5ede5ca3e9c92376576c6f8e9c96aa9f1a3d Mon Sep 17 00:00:00 2001 From: Matteo Silvestri Date: Mon, 27 Feb 2023 11:04:20 +0100 Subject: [PATCH 6/8] add GitlabOrgDiscoveryEntityProvider documentation Signed-off-by: Matteo Silvestri --- docs/integrations/gitlab/org.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 2ab72b4bd6..83dfd0f77d 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -31,4 +31,10 @@ catalog: yourProviderId: host: gitlab.com orgEnabled: true + group: org/teams # Optional. Must not end with slash. Accepts only groups under the provided path (excluded) + groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything ``` + +When the `group` parameter is provided, the corresponding path prefix will be stripped out from each matching group +when computing the unique entity name. e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will +be `avengers-gotg`. From 8f3ab173517d18b77945cf1f7374bb88ec0e7bce Mon Sep 17 00:00:00 2001 From: matteosilv Date: Mon, 27 Feb 2023 17:51:03 +0100 Subject: [PATCH 7/8] tidy up documentation Co-authored-by: Jamie Klassen Signed-off-by: matteosilv --- docs/integrations/gitlab/org.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 83dfd0f77d..db703d8036 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -31,7 +31,7 @@ catalog: yourProviderId: host: gitlab.com orgEnabled: true - group: org/teams # Optional. Must not end with slash. Accepts only groups under the provided path (excluded) + group: org/teams # Optional. Must not end with slash. Accepts only groups under the provided path (which will be stripped) groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything ``` From a7da2e1a5bdfce9a121e4b0eddf4a55a14c59020 Mon Sep 17 00:00:00 2001 From: matteosilv Date: Mon, 27 Feb 2023 17:51:27 +0100 Subject: [PATCH 8/8] fix typo Co-authored-by: Jamie Klassen Signed-off-by: matteosilv --- docs/integrations/gitlab/org.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index db703d8036..7ddc5202b5 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -32,7 +32,7 @@ catalog: host: gitlab.com orgEnabled: true group: org/teams # Optional. Must not end with slash. Accepts only groups under the provided path (which will be stripped) - groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything + groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything ``` When the `group` parameter is provided, the corresponding path prefix will be stripped out from each matching group