From 4a4b9ad2de274fdf47178edba54da6d74d858c31 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Aug 2022 11:51:33 +0200 Subject: [PATCH 1/9] Added project pattern for GitLab provider to be able to filter projects on more granular level. Signed-off-by: bnechyporenko --- plugins/catalog-backend-module-gitlab/src/lib/types.ts | 1 + .../src/providers/GitlabDiscoveryEntityProvider.ts | 4 ++++ .../catalog-backend-module-gitlab/src/providers/config.ts | 6 +++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 69de9d28c7..8a3c33c8f2 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -35,4 +35,5 @@ export type GitlabProviderConfig = { id: string; branch: string; catalogFile: string; + projectPattern: RegExp; }; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index 36c3ee90be..ffb123edf7 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -147,6 +147,10 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }; for await (const project of projects) { + if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) { + continue; + } + res.scanned++; if (project.archived) { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index d6ad28b2ca..a316ddc3c9 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -15,13 +15,14 @@ */ import { Config } from '@backstage/config'; -import { GitlabProviderConfig } from '../lib/types'; +import { GitlabProviderConfig } from '../lib'; /** * Extracts the gitlab config from a config object * * @public * + * @param id - The provider key * @param config - The config object to extract from */ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { @@ -30,6 +31,8 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { const branch = config.getOptionalString('branch') ?? 'master'; const catalogFile = config.getOptionalString('entityFilename') ?? 'catalog-info.yaml'; + const projectPattern = + new RegExp(config.getString('projectPattern')) ?? /[\s\S]*/; return { id, @@ -37,6 +40,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { branch, host, catalogFile, + projectPattern, }; } From 24979413a43493ff276862a29c23c078746522f5 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Aug 2022 11:56:43 +0200 Subject: [PATCH 2/9] Added project pattern for GitLab provider to be able to filter projects on more granular level. Signed-off-by: bnechyporenko --- .changeset/cool-months-tickle.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .changeset/cool-months-tickle.md diff --git a/.changeset/cool-months-tickle.md b/.changeset/cool-months-tickle.md new file mode 100644 index 0000000000..a160b13bd0 --- /dev/null +++ b/.changeset/cool-months-tickle.md @@ -0,0 +1,17 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': minor +--- + +Enhancing GitLab provider with filtering projects by pattern RegExp + +```yaml +providers: + gitlab: + stg: + host: gitlab.stg.company.io + branch: main + projectPattern: johndoe/ <== new option + entityFilename: template.yaml +``` + +With the abovementioned parameter you can filter projects, and keep only who belongs to the namespace "johndoe". From eae2f7c73224f1ed66bf856e4f4fa5f0a047a855 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Aug 2022 13:02:04 +0200 Subject: [PATCH 3/9] Added project pattern for GitLab provider to be able to filter projects on more granular level. Signed-off-by: bnechyporenko --- .changeset/cool-months-tickle.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/cool-months-tickle.md b/.changeset/cool-months-tickle.md index a160b13bd0..41ee67db33 100644 --- a/.changeset/cool-months-tickle.md +++ b/.changeset/cool-months-tickle.md @@ -10,8 +10,8 @@ providers: stg: host: gitlab.stg.company.io branch: main - projectPattern: johndoe/ <== new option + projectPattern: john/ <== new option entityFilename: template.yaml ``` -With the abovementioned parameter you can filter projects, and keep only who belongs to the namespace "johndoe". +With the aforementioned parameter you can filter projects, and keep only who belongs to the namespace "john". From 986fcb39d8dd2615ad39e26df9b4f0219854b62b Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Aug 2022 13:10:38 +0200 Subject: [PATCH 4/9] Added project pattern for GitLab provider to be able to filter projects on more granular level. Signed-off-by: bnechyporenko --- .../catalog-backend-module-gitlab/src/providers/config.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index a316ddc3c9..da2a7e0b8d 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -31,8 +31,9 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { const branch = config.getOptionalString('branch') ?? 'master'; const catalogFile = config.getOptionalString('entityFilename') ?? 'catalog-info.yaml'; - const projectPattern = - new RegExp(config.getString('projectPattern')) ?? /[\s\S]*/; + const projectPattern = new RegExp( + config.getString('projectPattern') ?? /[\s\S]*/, + ); return { id, From 77e84d6ea4113d664cb45caa7967f645dcc2bd67 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Aug 2022 13:18:03 +0200 Subject: [PATCH 5/9] Added project pattern for GitLab provider to be able to filter projects on more granular level. Signed-off-by: bnechyporenko --- .../catalog-backend-module-gitlab/src/providers/config.test.ts | 2 ++ plugins/catalog-backend-module-gitlab/src/providers/config.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index 892ce5f25c..cb36e5166c 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -52,6 +52,7 @@ describe('config', () => { branch: 'master', host: 'host', catalogFile: 'catalog-info.yaml', + projectPattern: /[\s\S]*/, }), ); }); @@ -81,6 +82,7 @@ describe('config', () => { branch: 'not-master', host: 'host', catalogFile: 'custom-file.yaml', + projectPattern: /[\s\S]*/, }), ); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index da2a7e0b8d..392922860f 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -32,7 +32,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { const catalogFile = config.getOptionalString('entityFilename') ?? 'catalog-info.yaml'; const projectPattern = new RegExp( - config.getString('projectPattern') ?? /[\s\S]*/, + config.getOptionalString('projectPattern') ?? /[\s\S]*/, ); return { From b169ddac671abcb583dafc010792b47a4462e979 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Aug 2022 13:23:12 +0200 Subject: [PATCH 6/9] Added project pattern for GitLab provider to be able to filter projects on more granular level. Signed-off-by: bnechyporenko --- docs/integrations/gitlab/discovery.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index aad73c8209..df5aadcab4 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -24,6 +24,7 @@ catalog: branch: main # Optional. Uses `master` as default group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole project will be scanned entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` + projectPattern: /[\s\S]*/ # Optional. Filters found projects based on provided patter. Defaults to `/[\s\S]*/`, what means to not filter anything ``` As this provider is not one of the default providers, you will first need to install From 76b4026d032060b356b44aea6582c0c003a030af Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Aug 2022 13:54:54 +0200 Subject: [PATCH 7/9] Added project pattern for GitLab provider to be able to filter projects on more granular level. Signed-off-by: bnechyporenko --- .../GitlabDiscoveryEntityProvider.test.ts | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index dc70772d3d..d4a13cf81e 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -227,4 +227,113 @@ describe('GitlabDiscoveryEntityProvider', () => { entities: expectedEntities, }); }); + + it('should filter found projects based on a provided project pattern', async () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + projectPattern: 'john/', + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + server.use( + rest.get( + `https://api.gitlab.example/api/v4/projects`, + (_req, res, ctx) => { + const response = [ + { + id: 123, + default_branch: 'master', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://api.gitlab.example/test-group/test-repo', + path_with_namespace: 'test-group/test-repo', + }, + { + id: 124, + default_branch: 'master', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://api.gitlab.example/john/example', + path_with_namespace: 'john/example', + }, + ]; + return res(ctx.json(response)); + }, + ), + rest.head( + 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', + (req, res, ctx) => { + if (req.url.searchParams.get('ref') === 'master') { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ), + rest.head( + 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', + (req, res, ctx) => { + if (req.url.searchParams.get('ref') === 'master') { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ), + ); + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', + 'backstage.io/managed-by-origin-location': + 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', + }, + name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9', + }, + spec: { + presence: 'optional', + target: + 'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }, + ], + }); + }); }); From 77f143585c84c8a155e7f2c0771a5ddaa5e1a38a Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Aug 2022 14:53:13 +0200 Subject: [PATCH 8/9] Incorporated a feedback Signed-off-by: bnechyporenko --- .changeset/cool-months-tickle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cool-months-tickle.md b/.changeset/cool-months-tickle.md index 41ee67db33..3c3370216e 100644 --- a/.changeset/cool-months-tickle.md +++ b/.changeset/cool-months-tickle.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend-module-gitlab': minor +'@backstage/plugin-catalog-backend-module-gitlab': patch --- Enhancing GitLab provider with filtering projects by pattern RegExp From 4890dc302563cfeafdedbd36d43db26eb00cf2ba Mon Sep 17 00:00:00 2001 From: Ben Lambert Date: Tue, 2 Aug 2022 15:05:17 +0200 Subject: [PATCH 9/9] Comment the new option Just update the changeset description a little bit Signed-off-by: Ben Lambert --- .changeset/cool-months-tickle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cool-months-tickle.md b/.changeset/cool-months-tickle.md index 3c3370216e..281c74bab0 100644 --- a/.changeset/cool-months-tickle.md +++ b/.changeset/cool-months-tickle.md @@ -10,7 +10,7 @@ providers: stg: host: gitlab.stg.company.io branch: main - projectPattern: john/ <== new option + projectPattern: 'john/' # new option entityFilename: template.yaml ```