Merge pull request #12933 from acierto/master

Enhancing GitLab provider with filtering projects by pattern RegExp

.changeset/cool-months-tickle.md

@@ -0,0 +1,17 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': patch
+---
+
+Enhancing GitLab provider with filtering projects by pattern RegExp
+
+```yaml
+providers:
+  gitlab:
+    stg:
+      host: gitlab.stg.company.io
+      branch: main
+      projectPattern: 'john/' # new option
+      entityFilename: template.yaml
+```
+
+With the aforementioned parameter you can filter projects, and keep only who belongs to the namespace "john".

docs/integrations/gitlab/discovery.md

@@ -24,6 +24,7 @@ catalog:
         branch: main # Optional. Uses `master` as default
         group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole project will be scanned
         entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
+        projectPattern: /[\s\S]*/ # Optional. Filters found projects based on provided patter. Defaults to `/[\s\S]*/`, what means to not filter anything
 ```
 
 As this provider is not one of the default providers, you will first need to install

plugins/catalog-backend-module-gitlab/src/lib/types.ts

@@ -35,4 +35,5 @@ export type GitlabProviderConfig = {
   id: string;
   branch: string;
   catalogFile: string;
+  projectPattern: RegExp;
 };

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts

@@ -227,4 +227,113 @@ describe('GitlabDiscoveryEntityProvider', () => {
       entities: expectedEntities,
     });
   });
+
+  it('should filter found projects based on a provided project pattern', async () => {
+    const config = new ConfigReader({
+      integrations: {
+        gitlab: [
+          {
+            host: 'test-gitlab',
+            apiBaseUrl: 'https://api.gitlab.example/api/v4',
+            token: '1234',
+          },
+        ],
+      },
+      catalog: {
+        providers: {
+          gitlab: {
+            'test-id': {
+              host: 'test-gitlab',
+              projectPattern: 'john/',
+            },
+          },
+        },
+      },
+    });
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    server.use(
+      rest.get(
+        `https://api.gitlab.example/api/v4/projects`,
+        (_req, res, ctx) => {
+          const response = [
+            {
+              id: 123,
+              default_branch: 'master',
+              archived: false,
+              last_activity_at: new Date().toString(),
+              web_url: 'https://api.gitlab.example/test-group/test-repo',
+              path_with_namespace: 'test-group/test-repo',
+            },
+            {
+              id: 124,
+              default_branch: 'master',
+              archived: false,
+              last_activity_at: new Date().toString(),
+              web_url: 'https://api.gitlab.example/john/example',
+              path_with_namespace: 'john/example',
+            },
+          ];
+          return res(ctx.json(response));
+        },
+      ),
+      rest.head(
+        'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
+        (req, res, ctx) => {
+          if (req.url.searchParams.get('ref') === 'master') {
+            return res(ctx.status(200));
+          }
+          return res(ctx.status(404, 'Not Found'));
+        },
+      ),
+      rest.head(
+        'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml',
+        (req, res, ctx) => {
+          if (req.url.searchParams.get('ref') === 'master') {
+            return res(ctx.status(200));
+          }
+          return res(ctx.status(404, 'Not Found'));
+        },
+      ),
+    );
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toBeCalledWith({
+      type: 'full',
+      entities: [
+        {
+          entity: {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'Location',
+            metadata: {
+              annotations: {
+                'backstage.io/managed-by-location':
+                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
+                'backstage.io/managed-by-origin-location':
+                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
+              },
+              name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9',
+            },
+            spec: {
+              presence: 'optional',
+              target:
+                'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
+              type: 'url',
+            },
+          },
+          locationKey: 'GitlabDiscoveryEntityProvider:test-id',
+        },
+      ],
+    });
+  });
 });

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts

@@ -147,6 +147,10 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
     };
 
     for await (const project of projects) {
+      if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) {
+        continue;
+      }
+
       res.scanned++;
 
       if (project.archived) {

plugins/catalog-backend-module-gitlab/src/providers/config.test.ts

@@ -52,6 +52,7 @@ describe('config', () => {
         branch: 'master',
         host: 'host',
         catalogFile: 'catalog-info.yaml',
+        projectPattern: /[\s\S]*/,
       }),
     );
   });
@@ -81,6 +82,7 @@ describe('config', () => {
         branch: 'not-master',
         host: 'host',
         catalogFile: 'custom-file.yaml',
+        projectPattern: /[\s\S]*/,
       }),
     );
   });

plugins/catalog-backend-module-gitlab/src/providers/config.ts

@@ -15,13 +15,14 @@
  */
 
 import { Config } from '@backstage/config';
-import { GitlabProviderConfig } from '../lib/types';
+import { GitlabProviderConfig } from '../lib';
 
 /**
  * Extracts the gitlab config from a config object
  *
  * @public
  *
+ * @param id - The provider key
  * @param config - The config object to extract from
  */
 function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
@@ -30,6 +31,9 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
   const branch = config.getOptionalString('branch') ?? 'master';
   const catalogFile =
     config.getOptionalString('entityFilename') ?? 'catalog-info.yaml';
+  const projectPattern = new RegExp(
+    config.getOptionalString('projectPattern') ?? /[\s\S]*/,
+  );
 
   return {
     id,
@@ -37,6 +41,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
     branch,
     host,
     catalogFile,
+    projectPattern,
   };
 }