Merge pull request #23373 from elaine-mattos/feat/add-event-support-gitlab-provider

Add event support for the Catalog Plugin GitLab Discovery Entity providers

.changeset/eleven-pandas-divide.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': patch
+---
+
+Added events support for `GitlabDiscoveryEntityProvider` and `GitlabOrgDiscoveryEntityProvider`.

.changeset/shy-students-clap.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab-org': patch
+---
+
+Added a new `catalog-backend-module-gitlab-org` module which adds the `GitlabOrgDiscoveryEntityProvider` to the catalog's providers using the new backend system.

docs/integrations/gitlab/discovery.md

@@ -11,28 +11,11 @@ entities from GitLab. The entity provider will crawl the GitLab instance and reg
 entities matching the configured paths. This can be useful as an alternative to
 static locations or manually adding things to the catalog.
 
-To use the discovery provider, you'll need a GitLab integration
-[set up](locations.md) with a `token`. Then you can add a provider config per group
-to the catalog configuration:
+This provider can also be configured to ingest GitLab data based on [GitLab Webhooks](https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#configure-a-webhook-in-gitlab). The events currently accepted are:
 
-```yaml
-catalog:
-  providers:
-    gitlab:
-      yourProviderId:
-        host: gitlab-host # Identifies one of the hosts set up in the integrations
-        branch: main # Optional. Used to discover on a specific branch
-        fallbackBranch: main # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default
-        skipForkedRepos: false # Optional. If the project is a fork, skip repository
-        group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned
-        entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
-        projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything
-        schedule: # optional; same options as in TaskScheduleDefinition
-          # supports cron, ISO duration, "human duration" as used in code
-          frequency: { minutes: 30 }
-          # supports ISO duration, "human duration" as used in code
-          timeout: { minutes: 3 }
-```
+- [`push`](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#push-events).
+
+## Installation
 
 As this provider is not one of the default providers, you will first need to install
 the gitlab catalog plugin:
@@ -42,15 +25,93 @@ the gitlab catalog plugin:
 yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab
 ```
 
-Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`:
+### Installation with New Backend System
+
+Then add the following to your backend initialization:
+
+```ts title="packages/backend/src/index.ts
+// optional if you want HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend/alpha'));
+// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha'));
+// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md
+// backend.add(eventsModuleGitlabEventRouter());
+// optional - token validator for the gitlab topic
+// backend.add(eventsModuleGitlabWebhook());
+backend.add(import('@backstage/plugin-catalog-backend-module-gitlab/alpha'));
+```
+
+You need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Further documentation:
+
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+### Installation with Legacy Backend System
+
+#### Installation without Events Support
+
+Add the segment below to `packages/backend/src/plugins/catalog.ts`:
 
 ```ts title="packages/backend/src/plugins/catalog.ts"
+/* highlight-add-next-line */
 import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
 
-const builder = await CatalogBuilder.create(env);
-/** ... other processors and/or providers ... */
-builder.addEntityProvider(
-  ...GitlabDiscoveryEntityProvider.fromConfig(env.config, {
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  /* highlight-add-start */
+  builder.addEntityProvider(
+    ...GitlabDiscoveryEntityProvider.fromConfig(env.config, {
+      logger: env.logger,
+      // optional: alternatively, use scheduler with schedule defined in app-config.yaml
+      schedule: env.scheduler.createScheduledTaskRunner({
+        frequency: { minutes: 30 },
+        timeout: { minutes: 3 },
+      }),
+      // optional: alternatively, use schedule
+      scheduler: env.scheduler,
+    }),
+  );
+  /* highlight-add-end */
+  // ..
+}
+```
+
+#### Installation with Events Support
+
+Please follow the installation instructions at
+
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+Additionally, you need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Set up your provider
+
+```ts title="packages/backend/src/plugins/catalog.ts"
+import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+/* highlight-add-next-line */
+import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
+import { Router } from 'express';
+import { PluginEnvironment } from '../types';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new ScaffolderEntitiesProcessor());
+  /* highlight-add-start */
+  const gitlabProvider = GitlabDiscoveryEntityProvider.fromConfig(env.config, {
     logger: env.logger,
     // optional: alternatively, use scheduler with schedule defined in app-config.yaml
     schedule: env.scheduler.createScheduledTaskRunner({
@@ -59,19 +120,45 @@ builder.addEntityProvider(
     }),
     // optional: alternatively, use schedule
     scheduler: env.scheduler,
-  }),
-);
+  });
+  env.eventBroker.subscribe(gitlabProvider);
+  builder.addEntityProvider(gitlabProvider);
+  /* highlight-add-end */
+  const { processingEngine, router } = await builder.build();
+  await processingEngine.start();
+  return router;
+}
+```
+
+## Configuration
+
+To use the discovery provider, you'll need a GitLab integration
+[set up](locations.md) with a `token`. Then you can add a provider config per group
+to the catalog configuration.
+
+> > NOTE: if you are using the New Backend System, the `schedule` has to be setup in the config, as shown below.
+
+```yaml title="app-config.yaml"
+catalog:
+  providers:
+    gitlab:
+      yourProviderId:
+        host: gitlab-host # Identifies one of the hosts set up in the integrations
+        branch: main # Optional. Used to discover on a specific branch
+        fallbackBranch: master # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default
+        skipForkedRepos: false # Optional. If the project is a fork, skip repository
+        group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned
+        entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
+        projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything
+        schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System
+          # supports cron, ISO duration, "human duration" as used in code
+          frequency: { minutes: 30 }
+          # supports ISO duration, "human duration" as used in code
+          timeout: { minutes: 3 }
 ```
 
 ## Alternative processor
 
-```yaml
-catalog:
-  locations:
-    - type: gitlab-discovery
-      target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml
-```
-
 As alternative to the entity provider `GitlabDiscoveryEntityProvider`
 you can still use the `GitLabDiscoveryProcessor`.
 
@@ -109,6 +196,15 @@ export default async function createPlugin(
 }
 ```
 
+And add the following to your app-config.yaml
+
+```yaml
+catalog:
+  locations:
+    - type: gitlab-discovery
+      target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml
+```
+
 If you don't want create location object if file with component definition do not exists in project, you can set the `skipReposWithoutExactFileMatch` option. That can reduce count of request to gitlab with 404 status code.
 
 If you don't want to create location object if the project is a fork, you can set the `skipForkedRepos` option.

docs/integrations/gitlab/org.md

@@ -11,34 +11,134 @@ groups -- directly from GitLab. The result is a hierarchy of
 [`Group`](../../features/software-catalog/descriptor-format.md#kind-group)
 entities that mirrors your org setup.
 
+This provider can also be configured to ingest GitLab data based on [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html). The events currently accepted are:
+
+- `group_create`
+- `group_destroy`
+- `group_rename`
+- `user_create`
+- `user_destroy`
+- `user_add_to_group`
+- `user_remove_from_group`
+
+## Installation
+
 As this provider is not one of the default providers, you will first need to install the Gitlab provider plugin:
 
 ```bash
 # From your Backstage root directory
-yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab
+yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab @backstage/plugin-catalog-backend-module-gitlab-org
 ```
 
-Then add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`:
+### Installation with New Backend System
+
+Then add the following to your backend initialization:
+
+```ts title="packages/backend/src/index.ts
+// optional if you want HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend/alpha'));
+// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha'));
+// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md
+// backend.add(eventsModuleGitlabEventRouter());
+// optional - token validator for the gitlab topic
+// backend.add(eventsModuleGitlabWebhook());
+backend.add(import('@backstage/plugin-catalog-backend-module-gitlab-org'));
+```
+
+You need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Further documentation:
+
+- [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html)
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+### Installation with Legacy Backend System
+
+#### Installation without Events Support
+
+Add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`:
 
 ```ts
 /* packages/backend/src/plugins/catalog.ts */
 /* highlight-add-next-line */
 import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
 
-const builder = await CatalogBuilder.create(env);
-/** ... other processors and/or providers ... */
-/* highlight-add-start */
-builder.addEntityProvider(
-  ...GitlabOrgDiscoveryEntityProvider.fromConfig(env.config, {
-    logger: env.logger,
-    // optional: alternatively, use scheduler with schedule defined in app-config.yaml
-    schedule: env.scheduler.createScheduledTaskRunner({
-      frequency: { minutes: 30 },
-      timeout: { minutes: 3 },
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  /** ... other processors and/or providers ... */
+  /* highlight-add-start */
+  builder.addEntityProvider(
+    ...GitlabOrgDiscoveryEntityProvider.fromConfig(env.config, {
+      logger: env.logger,
+      // optional: alternatively, use scheduler with schedule defined in app-config.yaml
+      schedule: env.scheduler.createScheduledTaskRunner({
+        frequency: { minutes: 30 },
+        timeout: { minutes: 3 },
+      }),
+      // optional: alternatively, use schedule
+      scheduler: env.scheduler,
     }),
-  }),
-);
-/* highlight-add-end */
+  );
+  /* highlight-add-end */
+  // ..
+}
+```
+
+#### Installation with Events Support
+
+Please follow the installation instructions at
+
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+Additionally, you need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Set up your provider
+
+```ts title="packages/backend/src/plugins/catalog.ts"
+import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+/* highlight-add-next-line */
+import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
+import { Router } from 'express';
+import { PluginEnvironment } from '../types';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new ScaffolderEntitiesProcessor());
+  /* highlight-add-start */
+  const gitlabOrgProvider = GitlabOrgDiscoveryEntityProvider.fromConfig(
+    env.config,
+    {
+      logger: env.logger,
+      // optional: alternatively, use scheduler with schedule defined in app-config.yaml
+      schedule: env.scheduler.createScheduledTaskRunner({
+        frequency: { minutes: 30 },
+        timeout: { minutes: 3 },
+      }),
+      // optional: alternatively, use schedule
+      scheduler: env.scheduler,
+    },
+  );
+  env.eventBroker.subscribe(gitlabOrgProvider);
+  builder.addEntityProvider(gitlabOrgProvider);
+  /* highlight-add-end */
+  const { processingEngine, router } = await builder.build();
+  await processingEngine.start();
+  return router;
+}
 ```
 
 ## Configuration
@@ -58,6 +158,8 @@ amount of data, this can take significant time and resources.
 The token used must have the `read_api` scope, and the Users and Groups fetched
 will be those visible to the account which provisioned the token.
 
+> > NOTE: if you are using the New Backend System, the `schedule` has to be setup in the config, as shown below.
+
 ```yaml
 catalog:
   providers:
@@ -66,7 +168,13 @@ catalog:
         host: gitlab.com
         orgEnabled: true
         group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped)
+        allowInherited: true # Allow groups to be ingested even if there are no direct members.
         groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything
+        schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System.
+          # supports cron, ISO duration, "human duration" as used in code
+          frequency: { minutes: 30 }
+          # supports ISO duration, "human duration" as used in code
+          timeout: { minutes: 3 }
 ```
 
 ### Groups
@@ -107,6 +215,74 @@ catalog:
           - allow: [Group, User]
 ```
 
+### Custom Transformers
+
+You can inject your own transformation logic to help map GitLab API responses
+into Backstage entities. You can do this on the user and group requests to
+enable you to do further processing or updates to these entities.
+
+To enable this you pass a function into the `GitlabOrgDiscoveryEntityProvider`. You can
+pass a `UserTransformer`, a `GroupEntitiesTransformer` or a `GroupNameTransformer` (or all of them)). The function is invoked
+for each item (user or group) that is returned from the API.
+
+The example below uses the groupNameTransformer option to change the metadata.name property of the Backstage Group Entity. Instead of populating it with the usual `group.full_path` data that comes from GitLab, it uses the `group.id`:
+
+```ts
+import { loggerToWinstonLogger } from '@backstage/backend-common';
+import {
+  coreServices,
+  createBackendModule,
+} from '@backstage/backend-plugin-api';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
+import {
+  GitlabOrgDiscoveryEntityProvider,
+  /* highlight-add-next-line */
+  GroupNameTransformerOptions,
+} from '@backstage/plugin-catalog-backend-module-gitlab';
+
+/* highlight-add-start */
+function customGroupNameTransformer(
+  options: GroupNameTransformerOptions,
+): string {
+  return `${options.group.id}`;
+}
+/* highlight-add-end */
+
+/**
+ * Registers the GitlabDiscoveryEntityProvider with the catalog processing extension point.
+ *
+ * @alpha
+ */
+export const catalogModuleGitlabOrgDiscoveryEntityProvider =
+  createBackendModule({
+    pluginId: 'catalog',
+    moduleId: 'gitlabOrgDiscoveryEntityProvider',
+    register(env) {
+      env.registerInit({
+        deps: {
+          config: coreServices.rootConfig,
+          catalog: catalogProcessingExtensionPoint,
+          logger: coreServices.logger,
+          scheduler: coreServices.scheduler,
+          events: eventsServiceRef,
+        },
+        async init({ config, catalog, logger, scheduler, events }) {
+          const gitlabOrgDiscoveryEntityProvider =
+            GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
+              /* highlight-add-next-line */
+              groupNameTransformer: customGroupNameTransformer,
+              logger: loggerToWinstonLogger(logger),
+              events,
+              scheduler,
+            });
+          catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider);
+        },
+      });
+    },
+  });
+```
+
 ## Troubleshooting
 
 **NOTE**: If any groups that are being ingested are empty groups (i.e. do not

plugins/catalog-backend-module-gitlab-org/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

plugins/catalog-backend-module-gitlab-org/README.md

@@ -0,0 +1,8 @@
+# Catalog Backend Module for GitLab Organizational Data
+
+This is an extension module to the plugin-catalog-backend plugin that provides group and user entities by scrapping or receiving events from a GitLab instance.
+
+## Getting started
+
+See [Backstage documentation](https://backstage.io/docs/integrations/gitlab/org) for details on how to install
+and configure the plugin.

plugins/catalog-backend-module-gitlab-org/api-report.md

@@ -0,0 +1,11 @@
+## API Report File for "@backstage/plugin-catalog-backend-module-gitlab-org"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { BackendFeature } from '@backstage/backend-plugin-api';
+
+// @public
+const catalogModuleGitlabOrgDiscoveryEntityProvider: () => BackendFeature;
+export default catalogModuleGitlabOrgDiscoveryEntityProvider;
+```

plugins/catalog-backend-module-gitlab-org/catalog-info.yaml

@@ -0,0 +1,10 @@
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: backstage-plugin-catalog-backend-module-gitlab-org
+  title: '@backstage/plugin-catalog-backend-module-gitlab-org'
+  description: The gitlab-org backend module for the catalog plugin.
+spec:
+  lifecycle: experimental
+  type: backstage-backend-plugin-module
+  owner: maintainers

plugins/catalog-backend-module-gitlab-org/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-gitlab-org",
+  "version": "0.0.0",
+  "description": "The gitlab-org backend module for the catalog plugin.",
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-backend-module-gitlab-org"
+  },
+  "license": "Apache-2.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "backstage-cli package build",
+    "clean": "backstage-cli package clean",
+    "lint": "backstage-cli package lint",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "start": "backstage-cli package start",
+    "test": "backstage-cli package test"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "workspace:^",
+    "@backstage/backend-plugin-api": "workspace:^",
+    "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^",
+    "@backstage/plugin-catalog-node": "workspace:^",
+    "@backstage/plugin-events-node": "workspace:^"
+  },
+  "devDependencies": {
+    "@backstage/backend-tasks": "workspace:^",
+    "@backstage/backend-test-utils": "workspace:^",
+    "@backstage/cli": "workspace:^",
+    "@backstage/plugin-events-backend-test-utils": "workspace:^",
+    "luxon": "^3.0.0"
+  }
+}

plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createServiceFactory } from '@backstage/backend-plugin-api';
+import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+import { mockServices, startTestBackend } from '@backstage/backend-test-utils';
+import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { TestEventsService } from '@backstage/plugin-events-backend-test-utils';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
+import { Duration } from 'luxon';
+import { catalogModuleGitlabOrgDiscoveryEntityProvider } from './catalogModuleGitlabOrgDiscoveryEntityProvider';
+
+describe('catalogModuleGitlabOrgDiscoveryEntityProvider', () => {
+  it('should register provider at the catalog extension point', async () => {
+    const events = new TestEventsService();
+    const eventsServiceFactory = createServiceFactory({
+      service: eventsServiceRef,
+      deps: {},
+      async factory({}) {
+        return events;
+      },
+    });
+    let addedProviders: Array<GitlabOrgDiscoveryEntityProvider> | undefined;
+    let usedSchedule: TaskScheduleDefinition | undefined;
+
+    const extensionPoint = {
+      addEntityProvider: (providers: any) => {
+        addedProviders = providers;
+      },
+    };
+    const connection = jest.fn() as unknown as EntityProviderConnection;
+    const runner = jest.fn();
+    const scheduler = mockServices.scheduler.mock({
+      createScheduledTaskRunner(schedule) {
+        usedSchedule = schedule;
+        return { run: runner };
+      },
+    });
+
+    const config = {
+      integrations: {
+        gitlab: [
+          {
+            host: 'test-gitlab',
+            apiBaseUrl: 'https://api.gitlab.example/api/v4',
+            token: '1234',
+          },
+        ],
+      },
+      catalog: {
+        providers: {
+          gitlab: {
+            'test-id': {
+              host: 'test-gitlab',
+              group: 'test-group',
+              orgEnabled: true,
+              schedule: {
+                frequency: 'P1M',
+                timeout: 'PT3M',
+              },
+            },
+          },
+        },
+      },
+    };
+
+    await startTestBackend({
+      extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
+      features: [
+        eventsServiceFactory(),
+        catalogModuleGitlabOrgDiscoveryEntityProvider(),
+        mockServices.rootConfig.factory({ data: config }),
+        mockServices.logger.factory(),
+        scheduler.factory,
+      ],
+    });
+
+    expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M'));
+    expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M'));
+    expect(addedProviders?.length).toEqual(1);
+    expect(runner).not.toHaveBeenCalled();
+
+    const provider = addedProviders!.pop()!;
+    expect(provider.getProviderName()).toEqual(
+      'GitlabOrgDiscoveryEntityProvider:test-id',
+    );
+    await provider.connect(connection);
+    expect(events.subscribed).toHaveLength(1);
+    expect(events.subscribed[0].id).toEqual(
+      'GitlabOrgDiscoveryEntityProvider:test-id',
+    );
+    expect(runner).toHaveBeenCalledTimes(1);
+  });
+});

plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { loggerToWinstonLogger } from '@backstage/backend-common';
+import {
+  coreServices,
+  createBackendModule,
+} from '@backstage/backend-plugin-api';
+import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
+
+/**
+ * Registers the GitlabOrgDiscoveryEntityProvider with the catalog processing extension point.
+ *
+ * @public
+ */
+export const catalogModuleGitlabOrgDiscoveryEntityProvider =
+  createBackendModule({
+    pluginId: 'catalog',
+    moduleId: 'gitlabOrgDiscoveryEntityProvider',
+    register(env) {
+      env.registerInit({
+        deps: {
+          config: coreServices.rootConfig,
+          catalog: catalogProcessingExtensionPoint,
+          logger: coreServices.logger,
+          scheduler: coreServices.scheduler,
+          events: eventsServiceRef,
+        },
+        async init({ config, catalog, logger, scheduler, events }) {
+          const gitlabOrgDiscoveryEntityProvider =
+            GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
+              logger: loggerToWinstonLogger(logger),
+              events,
+              scheduler,
+            });
+          catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider);
+        },
+      });
+    },
+  });

plugins/catalog-backend-module-gitlab-org/src/index.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The gitlab-org backend module for the catalog plugin.
+ *
+ * @packageDocumentation
+ */
+export { catalogModuleGitlabOrgDiscoveryEntityProvider as default } from './catalogModuleGitlabOrgDiscoveryEntityProvider';

plugins/catalog-backend-module-gitlab/CHANGELOG.md

@@ -385,8 +385,8 @@
 
   ref:
 
-  https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api
-  https://docs.gitlab.com/ee/api/members.html#limitations
+  <https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api>
+  <https://docs.gitlab.com/ee/api/members.html#limitations>
 
 - 890e3b5ad4: Make sure to include the error message when ingestion fails
 - 0b55f773a7: Removed some unused dependencies
@@ -417,8 +417,8 @@
 
   ref:
 
-  https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api
-  https://docs.gitlab.com/ee/api/members.html#limitations
+  <https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api>
+  <https://docs.gitlab.com/ee/api/members.html#limitations>
 
 - 0b55f773a7: Removed some unused dependencies
 - Updated dependencies
@@ -1211,7 +1211,7 @@
 - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
 
   Please find how to configure the schedule at the config at
-  https://backstage.io/docs/integrations/gitlab/discovery
+  <https://backstage.io/docs/integrations/gitlab/discovery>
 
 - 4c9f7847e4: Updated dependency `msw` to `^0.48.0` while moving it to be a dev dependency.
 - Updated dependencies
@@ -1250,7 +1250,7 @@
 - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
 
   Please find how to configure the schedule at the config at
-  https://backstage.io/docs/integrations/gitlab/discovery
+  <https://backstage.io/docs/integrations/gitlab/discovery>
 
 - Updated dependencies
   - @backstage/backend-common@0.16.0-next.0

plugins/catalog-backend-module-gitlab/api-report.md

@@ -8,6 +8,7 @@ import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node';
 import { Config } from '@backstage/config';
 import { EntityProvider } from '@backstage/plugin-catalog-node';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
+import { EventsService } from '@backstage/plugin-events-node';
 import { GitLabIntegrationConfig } from '@backstage/integration';
 import { GroupEntity } from '@backstage/catalog-model';
 import { LocationSpec } from '@backstage/plugin-catalog-node';
@@ -26,13 +27,13 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
     config: Config,
     options: {
       logger: LoggerService;
+      events?: EventsService;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
     },
   ): GitlabDiscoveryEntityProvider[];
   // (undocumented)
   getProviderName(): string;
-  // (undocumented)
   refresh(logger: LoggerService): Promise<void>;
 }
 
@@ -81,6 +82,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     config: Config,
     options: {
       logger: LoggerService;
+      events?: EventsService;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
       userTransformer?: UserTransformer;
@@ -103,6 +105,7 @@ export type GitlabProviderConfig = {
   projectPattern: RegExp;
   userPattern: RegExp;
   groupPattern: RegExp;
+  allowInherited?: boolean;
   orgEnabled?: boolean;
   schedule?: TaskScheduleDefinition;
   skipForkedRepos?: boolean;

plugins/catalog-backend-module-gitlab/package.json

@@ -1,18 +1,67 @@
 {
-  "name": "@backstage/plugin-catalog-backend-module-gitlab",
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "configSchema": "config.d.ts",
+  "dependencies": {
+    "@backstage/backend-common": "workspace:^",
+    "@backstage/backend-plugin-api": "workspace:^",
+    "@backstage/backend-tasks": "workspace:^",
+    "@backstage/catalog-model": "workspace:^",
+    "@backstage/config": "workspace:^",
+    "@backstage/integration": "workspace:^",
+    "@backstage/plugin-catalog-common": "workspace:^",
+    "@backstage/plugin-catalog-node": "workspace:^",
+    "@backstage/plugin-events-node": "workspace:^",
+    "@gitbeaker/rest": "^40.0.3",
+    "lodash": "^4.17.21",
+    "node-fetch": "^2.6.7",
+    "uuid": "^9.0.0"
+  },
   "description": "A Backstage catalog backend module that helps integrate towards GitLab",
-  "version": "0.3.15",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
-  "license": "Apache-2.0",
-  "publishConfig": {
-    "access": "public"
+  "devDependencies": {
+    "@backstage/backend-test-utils": "workspace:^",
+    "@backstage/cli": "workspace:^",
+    "@backstage/plugin-events-backend-test-utils": "workspace:^",
+    "@types/lodash": "^4.14.151",
+    "@types/uuid": "^9.0.0",
+    "luxon": "^3.0.0",
+    "msw": "^1.0.0"
   },
   "exports": {
     ".": "./src/index.ts",
     "./alpha": "./src/alpha.ts",
     "./package.json": "./package.json"
   },
+  "files": [
+    "config.d.ts",
+    "dist"
+  ],
+  "homepage": "https://backstage.io",
+  "keywords": [
+    "backstage"
+  ],
+  "license": "Apache-2.0",
+  "main": "src/index.ts",
+  "name": "@backstage/plugin-catalog-backend-module-gitlab",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "directory": "plugins/catalog-backend-module-gitlab",
+    "type": "git",
+    "url": "https://github.com/backstage/backstage"
+  },
+  "scripts": {
+    "build": "backstage-cli package build",
+    "clean": "backstage-cli package clean",
+    "lint": "backstage-cli package lint",
+    "postpack": "backstage-cli package postpack",
+    "prepack": "backstage-cli package prepack",
+    "start": "backstage-cli package start",
+    "test": "backstage-cli package test"
+  },
+  "types": "src/index.ts",
   "typesVersions": {
     "*": {
       "alpha": [
@@ -23,50 +72,5 @@
       ]
     }
   },
-  "backstage": {
-    "role": "backend-plugin-module"
-  },
-  "homepage": "https://backstage.io",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/backstage/backstage",
-    "directory": "plugins/catalog-backend-module-gitlab"
-  },
-  "keywords": [
-    "backstage"
-  ],
-  "scripts": {
-    "start": "backstage-cli package start",
-    "build": "backstage-cli package build",
-    "lint": "backstage-cli package lint",
-    "test": "backstage-cli package test",
-    "prepack": "backstage-cli package prepack",
-    "postpack": "backstage-cli package postpack",
-    "clean": "backstage-cli package clean"
-  },
-  "dependencies": {
-    "@backstage/backend-common": "workspace:^",
-    "@backstage/backend-plugin-api": "workspace:^",
-    "@backstage/backend-tasks": "workspace:^",
-    "@backstage/catalog-model": "workspace:^",
-    "@backstage/config": "workspace:^",
-    "@backstage/integration": "workspace:^",
-    "@backstage/plugin-catalog-node": "workspace:^",
-    "lodash": "^4.17.21",
-    "node-fetch": "^2.6.7",
-    "uuid": "^9.0.0"
-  },
-  "devDependencies": {
-    "@backstage/backend-test-utils": "workspace:^",
-    "@backstage/cli": "workspace:^",
-    "@types/lodash": "^4.14.151",
-    "@types/uuid": "^9.0.0",
-    "luxon": "^3.0.0",
-    "msw": "^1.0.0"
-  },
-  "files": [
-    "config.d.ts",
-    "dist"
-  ],
-  "configSchema": "config.d.ts"
+  "version": "0.3.15-next.1"
 }

plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts

@@ -0,0 +1,537 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { graphql, rest } from 'msw';
+import {
+  all_groups_response,
+  all_projects_response,
+  all_saas_users_response,
+  all_users_response,
+  apiBaseUrl,
+  apiBaseUrlSaas,
+  graphQlBaseUrl,
+  paged_endpoint,
+  saasGraphQlBaseUrl,
+  some_endpoint,
+  unhealthy_endpoint,
+  userID,
+} from './mocks';
+
+const httpHandlers = [
+  /**
+   * Project REST endpoint mocks
+   */
+
+  // fetch all projects in an instance
+  rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => {
+    return res(ctx.set('x-next-page', ''), ctx.json(all_projects_response));
+  }),
+
+  rest.get(`${apiBaseUrl}/projects/42`, (_, res, ctx) => {
+    return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' }));
+  }),
+
+  // testing non existing file
+  rest.get(
+    `${apiBaseUrl}/projects/test-group%2Ftest-repo1/repository/files/catalog-info.yaml`,
+    (_, res, ctx) => {
+      return res(ctx.status(400), ctx.json({ error: 'Not found' }));
+    },
+  ),
+
+  /**
+   * Group REST endpoint mocks
+   */
+
+  rest.get(`${apiBaseUrl}/groups`, (_req, res, ctx) => {
+    return res(ctx.set('x-next-page', ''), ctx.json(all_groups_response));
+  }),
+
+  rest.get(`${apiBaseUrl}/groups/42`, (_, res, ctx) => {
+    return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' }));
+  }),
+
+  rest.get(`${apiBaseUrlSaas}/groups/group1/members/all`, (_req, res, ctx) => {
+    return res(ctx.json(all_saas_users_response));
+  }),
+
+  /**
+   * Users REST endpoint mocks
+   */
+
+  rest.get(`${apiBaseUrl}/users`, (_req, res, ctx) => {
+    return res(ctx.set('x-next-page', ''), ctx.json(all_users_response));
+  }),
+
+  rest.get(`${apiBaseUrl}/users/${userID}`, (_, res, ctx) => {
+    return res(ctx.json(all_users_response.find(user => user.id === userID)));
+  }),
+
+  rest.get(`${apiBaseUrl}/users/42`, (_, res, ctx) => {
+    return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' }));
+  }),
+
+  /**
+   * others
+   */
+
+  // mock a 4 page response
+  rest.get(`${apiBaseUrl}${paged_endpoint}`, (req, res, ctx) => {
+    const page = req.url.searchParams.get('page');
+    const currentPage = page ? Number(page) : 1;
+    const fakePageCount = 4;
+
+    return res(
+      // set next page number header if page requested is less than count
+      ctx.set(
+        'x-next-page',
+        currentPage < fakePageCount ? String(currentPage + 1) : '',
+      ),
+      ctx.json([{ someContentOfPage: currentPage }]),
+    );
+  }),
+
+  rest.get(`${apiBaseUrl}${unhealthy_endpoint}`, (_, res, ctx) => {
+    return res(ctx.status(400), ctx.json({ error: 'some error' }));
+  }),
+
+  rest.get(`${apiBaseUrl}${some_endpoint}`, (req, res, ctx) => {
+    return res(ctx.json([{ endpoint: req.url.toString() }]));
+  }),
+];
+
+// dynamic handlers
+
+const httpGroupFindByIdDynamic = all_groups_response.map(group => {
+  return rest.get(`${apiBaseUrl}/groups/${group.id}`, (_, res, ctx) => {
+    return res(ctx.json(all_groups_response.find(g => g.id === group.id)));
+  });
+});
+
+const httpGroupListDescendantProjectsById = all_groups_response.map(group => {
+  return rest.get(
+    `${apiBaseUrl}/groups/${group.id}/projects`,
+    (_, res, ctx) => {
+      const projectsInGroup = all_projects_response.filter(p =>
+        p.path_with_namespace?.includes(group.name),
+      );
+
+      return res(ctx.json(projectsInGroup));
+    },
+  );
+});
+
+const httpGroupListDescendantProjectsByName = all_groups_response.map(group => {
+  return rest.get(
+    `${apiBaseUrl}/groups/${group.name}/projects`,
+    (_, res, ctx) => {
+      const projectsInGroup = all_projects_response.filter(p =>
+        p.path_with_namespace?.includes(group.name),
+      );
+
+      return res(ctx.json(projectsInGroup));
+    },
+  );
+});
+const httpGroupFindByNameDynamic = all_groups_response.map(group => {
+  return rest.get(`${apiBaseUrl}/groups/${group.name}`, (_, res, ctx) => {
+    return res(ctx.json(all_groups_response.find(g => g.name === group.name)));
+  });
+});
+const httpProjectFindByIdDynamic = all_projects_response.map(project => {
+  return rest.get(`${apiBaseUrl}/projects/${project.id}`, (_, res, ctx) => {
+    return res(ctx.json(all_projects_response.find(p => p.id === project.id)));
+  });
+});
+const httpProjectCatalogDynamic = all_projects_response.map(project => {
+  const path: string = project.path_with_namespace
+    ? project.path_with_namespace!.replace(/\//g, '%2F')
+    : `${project.path_with_namespace}%2F${project.name}`;
+
+  return rest.head(
+    `${apiBaseUrl}/projects/${path}/repository/files/catalog-info.yaml`,
+    (req, res, ctx) => {
+      const branch = req.url.searchParams.get('ref');
+      if (branch === project.default_branch) {
+        return res(ctx.status(200));
+      }
+      return res(ctx.status(404, 'Not Found'));
+    },
+  );
+});
+
+/**
+ * GraphQL endpoint mocks
+ */
+
+const graphqlHandlers = [
+  graphql
+    .link(graphQlBaseUrl)
+    .query('getGroupMembers', async (req, res, ctx) => {
+      const { group, relations } = req.variables;
+
+      if (group === 'group1' && relations.includes('DIRECT')) {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: [
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/1',
+                      username: 'user1',
+                      publicEmail: 'user1@example.com',
+                      name: 'user1',
+                      state: 'active',
+                      webUrl: 'user1.com',
+                      avatarUrl: 'user1',
+                    },
+                  },
+                ],
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+      // group with no associated members
+      if (group === 'group3' && relations.includes('DIRECT')) {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: [{}],
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+      if (group === 'saas-multi-user-group') {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: [
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/1',
+                      username: 'user1',
+                      publicEmail: 'user1@example.com',
+                      name: 'user1',
+                      state: 'active',
+                      webUrl: 'user1.com',
+                      avatarUrl: 'user1',
+                    },
+                  },
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/2',
+                      username: 'user2',
+                      publicEmail: 'user2@example.com',
+                      name: 'user2',
+                      state: 'active',
+                      webUrl: 'user2.com',
+                      avatarUrl: 'user2',
+                    },
+                  },
+                ],
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'non-existing-group' || group === '') {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'multi-page' && relations.includes('DIRECT')) {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: req.variables.endCursor
+                  ? [{ user: { id: 'gid://gitlab/User/2' } }]
+                  : [{ user: { id: 'gid://gitlab/User/1' } }],
+                pageInfo: {
+                  endCursor: req.variables.endCursor ? 'end' : 'next',
+                  hasNextPage: !req.variables.endCursor,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'multi-page-saas') {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: req.variables.endCursor
+                  ? [
+                      {
+                        user: {
+                          id: 'gid://gitlab/User/1',
+                          username: 'user1',
+                          publicEmail: 'user1@example.com',
+                          name: 'user1',
+                          state: 'active',
+                          webUrl: 'user1.com',
+                          avatarUrl: 'user1',
+                        },
+                      },
+                    ]
+                  : [
+                      {
+                        user: {
+                          id: 'gid://gitlab/User/2',
+                          username: 'user2',
+                          publicEmail: 'user2@example.com',
+                          name: 'user2',
+                          state: 'active',
+                          webUrl: 'user2.com',
+                          avatarUrl: 'user2',
+                        },
+                      },
+                    ],
+                pageInfo: {
+                  endCursor: req.variables.endCursor ? 'end' : 'next',
+                  hasNextPage: !req.variables.endCursor,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'error-group') {
+        return res(
+          ctx.errors([
+            { message: 'Unexpected end of document', locations: [] },
+          ]),
+        );
+      }
+
+      return res(ctx.status(200));
+    }),
+
+  graphql
+    .link(graphQlBaseUrl)
+    .query('listDescendantGroups', async (req, res, ctx) => {
+      const { group } = req.variables;
+
+      if (group === 'error-group') {
+        return res(
+          ctx.errors([
+            { message: 'Unexpected end of document', locations: [] },
+          ]),
+        );
+      }
+
+      if (group === 'group-with-parent') {
+        return res(
+          ctx.data({
+            group: {
+              descendantGroups: {
+                nodes: [
+                  {
+                    id: 'gid://gitlab/Group/1',
+                    name: 'group-with-parent',
+                    description: 'description1',
+                    fullPath: 'path/group-with-parent',
+                    parent: {
+                      id: '123',
+                    },
+                  },
+                ],
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+      if (group === 'root') {
+        return res(
+          ctx.data({
+            group: {
+              descendantGroups: {
+                nodes: req.variables.endCursor
+                  ? [
+                      {
+                        id: 'gid://gitlab/Group/1',
+                        name: 'group1',
+                        description: 'description1',
+                        fullPath: 'path/group1',
+                        parent: {
+                          id: '123',
+                        },
+                      },
+                    ]
+                  : [
+                      {
+                        id: 'gid://gitlab/Group/2',
+                        name: 'group2',
+                        description: 'description2',
+                        fullPath: 'path/group2',
+                        parent: {
+                          id: '123',
+                        },
+                      },
+                    ],
+                pageInfo: {
+                  endCursor: req.variables.endCursor ? 'end' : 'next',
+                  hasNextPage: !req.variables.endCursor,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'non-existing-group') {
+        return res(
+          ctx.data({
+            group: {},
+          }),
+        );
+      }
+      return res(ctx.status(200));
+    }),
+
+  graphql
+    .link(saasGraphQlBaseUrl)
+    .query('getGroupMembers', async (req, res, ctx) => {
+      const { group } = req.variables;
+
+      if (group === 'saas-multi-user-group') {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: [
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/1',
+                      username: 'user1',
+                      publicEmail: 'user1@example.com',
+                      name: 'user1',
+                      state: 'active',
+                      webUrl: 'user1.com',
+                      avatarUrl: 'user1',
+                    },
+                  },
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/2',
+                      username: 'user2',
+                      publicEmail: 'user2@example.com',
+                      name: 'user2',
+                      state: 'active',
+                      webUrl: 'user2.com',
+                      avatarUrl: 'user2',
+                    },
+                  },
+                ],
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === '') {
+        return res(ctx.data({}));
+      }
+
+      if (group === 'error-group') {
+        return res(
+          ctx.errors([
+            { message: 'Unexpected end of document', locations: [] },
+          ]),
+        );
+      }
+
+      return res(ctx.status(200));
+    }),
+
+  graphql
+    .link(saasGraphQlBaseUrl)
+    .query('listDescendantGroups', async (_, res, ctx) => {
+      return res(
+        ctx.data({
+          group: {
+            descendantGroups: {
+              nodes: [
+                {
+                  id: 'gid://gitlab/Group/456',
+                  name: 'group1',
+                  description: 'Group1',
+                  fullPath: 'group1/group1',
+                  parent: {
+                    id: 'gid://gitlab/Group/123',
+                  },
+                },
+              ],
+              pageInfo: {
+                endCursor: 'end',
+                hasNextPage: false,
+              },
+            },
+          },
+        }),
+      );
+    }),
+];
+
+export const handlers = [
+  ...httpHandlers,
+  ...httpProjectFindByIdDynamic,
+  ...httpProjectCatalogDynamic,
+  ...httpGroupFindByIdDynamic,
+  ...httpGroupFindByNameDynamic,
+  ...httpGroupListDescendantProjectsById,
+  ...httpGroupListDescendantProjectsByName,
+  ...graphqlHandlers,
+];

plugins/catalog-backend-module-gitlab/src/lib/client.ts

@@ -24,6 +24,7 @@ import {
   GitLabDescendantGroupsResponse,
   GitLabGroup,
   GitLabGroupMembersResponse,
+  GitLabProject,
   GitLabUser,
   PagedResponse,
 } from './types';
@@ -80,6 +81,39 @@ export class GitLabClient {
     return this.pagedRequest(`/projects`, options);
   }
 
+  async getProjectById(
+    projectId: number,
+    options?: CommonListOptions,
+  ): Promise<GitLabProject> {
+    // Make the request to the GitLab API
+    const response = await this.nonPagedRequest(
+      `/projects/${projectId}`,
+      options,
+    );
+
+    return response;
+  }
+
+  async getGroupById(
+    groupId: number,
+    options?: CommonListOptions,
+  ): Promise<GitLabGroup> {
+    // Make the request to the GitLab API
+    const response = await this.nonPagedRequest(`/groups/${groupId}`, options);
+
+    return response;
+  }
+
+  async getUserById(
+    userId: number,
+    options?: CommonListOptions,
+  ): Promise<GitLabUser> {
+    // Make the request to the GitLab API
+    const response = await this.nonPagedRequest(`/users/${userId}`, options);
+
+    return response;
+  }
+
   async listUsers(
     options?: UserListOptions,
   ): Promise<PagedResponse<GitLabUser>> {
@@ -325,9 +359,13 @@ export class GitLabClient {
     options?: CommonListOptions,
   ): Promise<PagedResponse<T>> {
     const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
+
     for (const key in options) {
-      if (options[key] !== undefined && options[key] !== '') {
-        request.searchParams.append(key, options[key]!.toString());
+      if (options.hasOwnProperty(key)) {
+        const value = options[key];
+        if (value !== undefined && value !== '') {
+          request.searchParams.append(key, value.toString());
+        }
       }
     }
 
@@ -336,6 +374,7 @@ export class GitLabClient {
       request.toString(),
       getGitLabRequestOptions(this.config),
     );
+
     if (!response.ok) {
       throw new Error(
         `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
@@ -343,6 +382,7 @@ export class GitLabClient {
         } - ${response.statusText}`,
       );
     }
+
     return response.json().then(items => {
       const nextPage = response.headers.get('x-next-page');
 
@@ -352,6 +392,37 @@ export class GitLabClient {
       } as PagedResponse<any>;
     });
   }
+
+  async nonPagedRequest<T = any>(
+    endpoint: string,
+    options?: CommonListOptions,
+  ): Promise<T> {
+    const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
+
+    for (const key in options) {
+      if (options.hasOwnProperty(key)) {
+        const value = options[key];
+        if (value !== undefined && value !== '') {
+          request.searchParams.append(key, value.toString());
+        }
+      }
+    }
+
+    const response = await fetch(
+      request.toString(),
+      getGitLabRequestOptions(this.config),
+    );
+
+    if (!response.ok) {
+      throw new Error(
+        `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
+          response.status
+        } - ${response.statusText}`,
+      );
+    }
+
+    return response.json();
+  }
 }
 
 /**

plugins/catalog-backend-module-gitlab/src/lib/index.ts

@@ -14,14 +14,15 @@
  * limitations under the License.
  */
 
+export { readGitlabConfigs } from '../providers/config';
 export { GitLabClient, paginated } from './client';
 export type {
-  GitLabUser,
   GitLabGroup,
   GitLabGroupSamlIdentity,
   GitLabProject,
-  GitlabProviderConfig,
+  GitLabUser,
   GitlabGroupDescription,
+  GitlabProviderConfig,
   GroupNameTransformer,
   GroupNameTransformerOptions,
   GroupTransformer,
@@ -29,4 +30,3 @@ export type {
   UserTransformer,
   UserTransformerOptions,
 } from './types';
-export { readGitlabConfigs } from '../providers/config';

plugins/catalog-backend-module-gitlab/src/lib/types.ts

@@ -34,6 +34,9 @@ export type GitlabProjectForkedFrom = {
 
 export type GitLabProject = {
   id: number;
+  description?: string;
+  name?: string;
+  path?: string;
   default_branch?: string;
   archived: boolean;
   last_activity_at: string;
@@ -180,6 +183,11 @@ export type GitlabProviderConfig = {
    */
   groupPattern: RegExp;
 
+  /**
+   * If true, the provider will also ingest add inherited users to the ingested groups
+   */
+  allowInherited?: boolean;
+
   orgEnabled?: boolean;
   schedule?: TaskScheduleDefinition;
   /**
@@ -242,3 +250,72 @@ export interface GroupTransformerOptions {
   providerConfig: GitlabProviderConfig;
   groupNameTransformer: GroupNameTransformer;
 }
+
+/**
+ * Represents the schema for system hook events related to groups.
+ * https://docs.gitlab.com/ee/administration/system_hooks.html
+ *
+ * @public
+ */
+export type SystemHookBaseGroupEventsSchema = {
+  created_at: string;
+  updated_at: string;
+  name: string;
+  path: string;
+  full_path: string;
+  group_id: number;
+};
+
+/**
+ * Represents the schema for system hook events related to users.
+ * https://docs.gitlab.com/ee/administration/system_hooks.html
+ *
+ * @public
+ */
+export type SystemHookBaseUserEventsSchema = {
+  created_at: string;
+  updated_at: string;
+  email: string;
+  name: string;
+  username: string;
+  user_id: number;
+};
+
+/**
+ * Represents the schema for system hook events related to user memberships.
+ * https://docs.gitlab.com/ee/administration/system_hooks.html
+ *
+ * @public
+ */
+export type SystemHookBaseMembershipEventsSchema = {
+  created_at: string;
+  updated_at: string;
+  group_name: string;
+  group_path: string;
+  group_id: number;
+  user_username: string;
+  user_email: string;
+  user_name: string;
+  user_id: number;
+  group_access: string;
+};
+
+/**
+ * Represents the schema for system hook events related to projects.
+ * https://docs.gitlab.com/ee/administration/system_hooks.html
+ *
+ * @public
+ */
+export type SystemHookBaseProjectEventsSchema = {
+  created_at: string;
+  updated_at: string;
+  event_name: string;
+  name: string;
+  owner_email: string;
+  owner_name: string;
+  owners: { name: string; email: string }[];
+  path: string;
+  path_with_namespace: string;
+  project_id: number;
+  project_visibility: string;
+};

plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts

@@ -14,15 +14,27 @@
  * limitations under the License.
  */
 
+import { createServiceFactory } from '@backstage/backend-plugin-api';
 import { TaskScheduleDefinition } from '@backstage/backend-tasks';
 import { mockServices, startTestBackend } from '@backstage/backend-test-utils';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
 import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { TestEventsService } from '@backstage/plugin-events-backend-test-utils';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
 import { Duration } from 'luxon';
-import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider';
 import { GitlabDiscoveryEntityProvider } from '../providers';
+import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider';
 
 describe('catalogModuleGitlabDiscoveryEntityProvider', () => {
   it('should register provider at the catalog extension point', async () => {
+    const events = new TestEventsService();
+    const eventsServiceFactory = createServiceFactory({
+      service: eventsServiceRef,
+      deps: {},
+      async factory({}) {
+        return events;
+      },
+    });
     let addedProviders: Array<GitlabDiscoveryEntityProvider> | undefined;
     let usedSchedule: TaskScheduleDefinition | undefined;
 
@@ -31,6 +43,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => {
         addedProviders = providers;
       },
     };
+    const connection = jest.fn() as unknown as EntityProviderConnection;
     const runner = jest.fn();
     const scheduler = mockServices.scheduler.mock({
       createScheduledTaskRunner(schedule) {
@@ -68,6 +81,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => {
     await startTestBackend({
       extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
       features: [
+        eventsServiceFactory(),
         catalogModuleGitlabDiscoveryEntityProvider(),
         mockServices.rootConfig.factory({ data: config }),
         mockServices.logger.factory(),
@@ -78,9 +92,17 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => {
     expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M'));
     expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M'));
     expect(addedProviders?.length).toEqual(1);
-    expect(addedProviders?.pop()?.getProviderName()).toEqual(
+    expect(runner).not.toHaveBeenCalled();
+
+    const provider = addedProviders!.pop()!;
+    expect(provider.getProviderName()).toEqual(
       'GitlabDiscoveryEntityProvider:test-id',
     );
-    expect(runner).not.toHaveBeenCalled();
+    await provider.connect(connection);
+    expect(events.subscribed).toHaveLength(1);
+    expect(events.subscribed[0].id).toEqual(
+      'GitlabDiscoveryEntityProvider:test-id',
+    );
+    expect(runner).toHaveBeenCalledTimes(1);
   });
 });

plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts

@@ -19,6 +19,7 @@ import {
   createBackendModule,
 } from '@backstage/backend-plugin-api';
 import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
 import { GitlabDiscoveryEntityProvider } from '../providers';
 
 /**
@@ -26,6 +27,7 @@ import { GitlabDiscoveryEntityProvider } from '../providers';
  *
  * @alpha
  */
+
 export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({
   pluginId: 'catalog',
   moduleId: 'gitlab-discovery-entity-provider',
@@ -36,14 +38,16 @@ export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({
         catalog: catalogProcessingExtensionPoint,
         logger: coreServices.logger,
         scheduler: coreServices.scheduler,
+        events: eventsServiceRef,
       },
-      async init({ config, catalog, logger, scheduler }) {
-        catalog.addEntityProvider(
+      async init({ config, catalog, logger, scheduler, events }) {
+        const gitlabDiscoveryEntityProvider =
           GitlabDiscoveryEntityProvider.fromConfig(config, {
             logger,
+            events,
             scheduler,
-          }),
-        );
+          });
+        catalog.addEntityProvider(gitlabDiscoveryEntityProvider);
       },
     });
   },

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts

@@ -23,10 +23,16 @@ import {
 import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
 import { ConfigReader } from '@backstage/config';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
-import { rest } from 'msw';
+import { DefaultEventsService } from '@backstage/plugin-events-node';
 import { setupServer } from 'msw/node';
+import { handlers } from '../__testUtils__/handlers';
+import * as mock from '../__testUtils__/mocks';
 import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider';
 
+const server = setupServer(...handlers);
+setupRequestMockHandlers(server);
+afterEach(() => jest.resetAllMocks());
+
 class PersistingTaskRunner implements TaskRunner {
   private tasks: TaskInvocationDefinition[] = [];
 
@@ -42,13 +48,8 @@ class PersistingTaskRunner implements TaskRunner {
 
 const logger = getVoidLogger();
 
-const server = setupServer();
-
-describe('GitlabDiscoveryEntityProvider', () => {
-  setupRequestMockHandlers(server);
-  afterEach(() => jest.resetAllMocks());
-
-  it('no provider config', () => {
+describe('GitlabDiscoveryEntityProvider - configuration', () => {
+  it('should not instantiate providers when no config found', () => {
     const schedule = new PersistingTaskRunner();
     const config = new ConfigReader({});
     const providers = GitlabDiscoveryEntityProvider.fromConfig(config, {
@@ -59,29 +60,46 @@ describe('GitlabDiscoveryEntityProvider', () => {
     expect(providers).toHaveLength(0);
   });
 
-  it('single simple discovery config', () => {
+  it('should fail without schedule nor scheduler', () => {
+    const config = new ConfigReader(mock.config_single_integration_branch);
+
+    expect(() =>
+      GitlabDiscoveryEntityProvider.fromConfig(config, {
+        logger,
+      }),
+    ).toThrow('Either schedule or scheduler must be provided');
+  });
+
+  it('should fail with scheduler but no schedule config', () => {
+    const scheduler = {
+      createScheduledTaskRunner: (_: any) => jest.fn(),
+    } as unknown as PluginTaskScheduler;
+    const config = new ConfigReader(mock.config_no_schedule_integration);
+
+    expect(() =>
+      GitlabDiscoveryEntityProvider.fromConfig(config, {
+        logger,
+        scheduler,
+      }),
+    ).toThrow(
+      'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id',
+    );
+  });
+  it('should throw error when no matching GitLab integration config found', () => {
     const schedule = new PersistingTaskRunner();
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(mock.config_github_host);
+
+    expect(() => {
+      GitlabDiscoveryEntityProvider.fromConfig(config, {
+        logger,
+        schedule,
+      });
+    }).toThrow('No gitlab integration found that matches host example.com');
+  });
+
+  it('should instantiate provider with single simple discovery config', () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader(mock.config_single_integration_branch);
     const providers = GitlabDiscoveryEntityProvider.fromConfig(config, {
       logger,
       schedule,
@@ -93,33 +111,9 @@ describe('GitlabDiscoveryEntityProvider', () => {
     );
   });
 
-  it('multiple discovery configs', () => {
+  it('should instantiate providers when multiple discovery configs', () => {
     const schedule = new PersistingTaskRunner();
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-            'second-test': {
-              host: 'test-gitlab',
-              group: 'second-group',
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(mock.config_double_integration);
     const providers = GitlabDiscoveryEntityProvider.fromConfig(config, {
       logger,
       schedule,
@@ -133,29 +127,10 @@ describe('GitlabDiscoveryEntityProvider', () => {
       'GitlabDiscoveryEntityProvider:second-test',
     );
   });
-
-  it('apply full update on scheduled execution', async () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-          },
-        },
-      },
-    });
+});
+describe('GitlabDiscoveryEntityProvider - refresh', () => {
+  it('should apply full update on scheduled execution', async () => {
+    const config = new ConfigReader(mock.config_no_org_integration);
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
@@ -169,92 +144,28 @@ describe('GitlabDiscoveryEntityProvider', () => {
       'GitlabDiscoveryEntityProvider:test-id',
     );
 
-    server.use(
-      rest.get(
-        `https://api.gitlab.example/api/v4/groups/test-group/projects`,
-        (_req, res, ctx) => {
-          const response = [
-            {
-              id: 123,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/test-group/test-repo',
-              path_with_namespace: 'test-group/test-repo',
-            },
-          ];
-          return res(ctx.json(response));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-    );
-
     await provider.connect(entityProviderConnection);
 
     const taskDef = schedule.getTasks()[0];
     expect(taskDef.id).toEqual('GitlabDiscoveryEntityProvider:test-id:refresh');
     await (taskDef.fn as () => Promise<void>)();
 
-    const url = `https://api.gitlab.example/test-group/test-repo/-/blob/master/catalog-info.yaml`;
-    const expectedEntities = [
-      {
-        entity: {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'Location',
-          metadata: {
-            annotations: {
-              'backstage.io/managed-by-location': `url:${url}`,
-              'backstage.io/managed-by-origin-location': `url:${url}`,
-            },
-            name: 'generated-cd37bf72a2fe92603f4255d9f49c6c1ead746a48',
-          },
-          spec: {
-            presence: 'optional',
-            target: `${url}`,
-            type: 'url',
-          },
-        },
-        locationKey: 'GitlabDiscoveryEntityProvider:test-id',
-      },
-    ];
-
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
       type: 'full',
-      entities: expectedEntities,
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
     });
   });
 
   it('should filter found projects based on a provided project pattern', async () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              projectPattern: 'john/',
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(
+      mock.config_single_integration_project_pattern,
+    );
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
@@ -265,50 +176,10 @@ describe('GitlabDiscoveryEntityProvider', () => {
       schedule,
     })[0];
 
-    server.use(
-      rest.get(
-        `https://api.gitlab.example/api/v4/projects`,
-        (_req, res, ctx) => {
-          const response = [
-            {
-              id: 123,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/test-group/test-repo',
-              path_with_namespace: 'test-group/test-repo',
-            },
-            {
-              id: 124,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/john/example',
-              path_with_namespace: 'john/example',
-            },
-          ];
-          return res(ctx.json(response));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-    );
+    const projectPattern =
+      mock.config_single_integration_project_pattern.catalog.providers.gitlab[
+        'test-id'
+      ].projectPattern;
 
     await provider.connect(entityProviderConnection);
 
@@ -316,55 +187,20 @@ describe('GitlabDiscoveryEntityProvider', () => {
 
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
       type: 'full',
-      entities: [
-        {
-          entity: {
-            apiVersion: 'backstage.io/v1alpha1',
-            kind: 'Location',
-            metadata: {
-              annotations: {
-                'backstage.io/managed-by-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-                'backstage.io/managed-by-origin-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-              },
-              name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9',
-            },
-            spec: {
-              presence: 'optional',
-              target:
-                'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-              type: 'url',
-            },
-          },
-          locationKey: 'GitlabDiscoveryEntityProvider:test-id',
-        },
-      ],
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes(projectPattern) &&
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
     });
   });
 
   it('should filter fork projects', async () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              skipForkedRepos: true,
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(mock.config_single_integration_skip_forks);
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
@@ -375,216 +211,26 @@ describe('GitlabDiscoveryEntityProvider', () => {
       schedule,
     })[0];
 
-    server.use(
-      rest.get(
-        `https://api.gitlab.example/api/v4/projects`,
-        (_req, res, ctx) => {
-          const response = [
-            {
-              id: 123,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/test-group/test-repo',
-              path_with_namespace: 'test-group/test-repo',
-              forked_from_project: {
-                id: 13083,
-              },
-            },
-            {
-              id: 124,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/john/example',
-              path_with_namespace: 'john/example',
-            },
-          ];
-          return res(ctx.json(response));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-    );
-
     await provider.connect(entityProviderConnection);
 
     await provider.refresh(logger);
 
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
       type: 'full',
-      entities: [
-        {
-          entity: {
-            apiVersion: 'backstage.io/v1alpha1',
-            kind: 'Location',
-            metadata: {
-              annotations: {
-                'backstage.io/managed-by-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-                'backstage.io/managed-by-origin-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-              },
-              name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9',
-            },
-            spec: {
-              presence: 'optional',
-              target:
-                'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-              type: 'url',
-            },
-          },
-          locationKey: 'GitlabDiscoveryEntityProvider:test-id',
-        },
-      ],
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('forked') &&
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
     });
   });
 
-  it('fail without schedule and scheduler', () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-          },
-        },
-      },
-    });
-
-    expect(() =>
-      GitlabDiscoveryEntityProvider.fromConfig(config, {
-        logger,
-      }),
-    ).toThrow('Either schedule or scheduler must be provided');
-  });
-
-  it('fail with scheduler but no schedule config', () => {
-    const scheduler = {
-      createScheduledTaskRunner: (_: any) => jest.fn(),
-    } as unknown as PluginTaskScheduler;
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-          },
-        },
-      },
-    });
-
-    expect(() =>
-      GitlabDiscoveryEntityProvider.fromConfig(config, {
-        logger,
-        scheduler,
-      }),
-    ).toThrow(
-      'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id',
-    );
-  });
-
-  it('single simple provider config with schedule in config', async () => {
-    const schedule = new PersistingTaskRunner();
-    const scheduler = {
-      createScheduledTaskRunner: (_: any) => schedule,
-    } as unknown as PluginTaskScheduler;
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-              schedule: {
-                frequency: 'PT30M',
-                timeout: 'PT3M',
-              },
-            },
-          },
-        },
-      },
-    });
-    const providers = GitlabDiscoveryEntityProvider.fromConfig(config, {
-      logger,
-      scheduler,
-    });
-
-    expect(providers).toHaveLength(1);
-    expect(providers[0].getProviderName()).toEqual(
-      'GitlabDiscoveryEntityProvider:test-id',
-    );
-  });
-
   it('should filter found projects based on the branch', async () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              branch: 'test',
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(mock.config_single_integration_branch);
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
@@ -595,47 +241,9 @@ describe('GitlabDiscoveryEntityProvider', () => {
       schedule,
     })[0];
 
-    server.use(
-      rest.get(
-        `https://api.gitlab.example/api/v4/projects`,
-        (_req, res, ctx) => {
-          const response = [
-            {
-              id: 123,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/test-group/test-repo',
-              path_with_namespace: 'test-group/test-repo',
-            },
-            {
-              id: 124,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/john/example',
-              path_with_namespace: 'john/example',
-            },
-          ];
-          return res(ctx.json(response));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
-        (_, res, ctx) => {
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'test') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-    );
+    const configured_branch =
+      mock.config_single_integration_branch.catalog.providers.gitlab['test-id']
+        .branch;
 
     await provider.connect(entityProviderConnection);
 
@@ -643,30 +251,229 @@ describe('GitlabDiscoveryEntityProvider', () => {
 
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
       type: 'full',
-      entities: [
-        {
-          entity: {
-            apiVersion: 'backstage.io/v1alpha1',
-            kind: 'Location',
-            metadata: {
-              annotations: {
-                'backstage.io/managed-by-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml',
-                'backstage.io/managed-by-origin-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml',
-              },
-              name: 'generated-232185d858fee049986d202c10316d634e76a3d1',
-            },
-            spec: {
-              presence: 'optional',
-              target:
-                'https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml',
-              type: 'url',
-            },
-          },
-          locationKey: 'GitlabDiscoveryEntityProvider:test-id',
-        },
-      ],
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes(configured_branch) &&
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
+    });
+  });
+
+  it('should only include projects with fallback branch', async () => {
+    const config = new ConfigReader(mock.config_fallbackBranch_branch);
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const configured_branch =
+      mock.config_fallbackBranch_branch.catalog.providers.gitlab['test-id']
+        .fallbackBranch;
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes(configured_branch) &&
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
+    });
+  });
+
+  it('should ignore projects outside group scope', async () => {
+    const config = new ConfigReader(mock.config_single_integration_group);
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const configured_group =
+      mock.config_single_integration_group.catalog.providers.gitlab['test-id']
+        .group;
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: mock.expected_location_entities.filter(entity =>
+        entity.entity.metadata.annotations[
+          'backstage.io/managed-by-location'
+        ].includes(configured_group),
+      ),
     });
   });
 });
+describe('GitlabDiscoveryEntityProvider - events', () => {
+  it('should ignore push event if project is forked', async () => {
+    const config = new ConfigReader(mock.config_single_integration_skip_forks);
+
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_add_event_forked);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+
+  it(`should skip refresh and mutation when project pattern doesn't match`, async () => {
+    const config = new ConfigReader(mock.config_unmatched_project_integration);
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_add_event);
+
+    expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+
+  it('should ignore projects outside group scope', async () => {
+    const config = new ConfigReader(mock.config_single_integration_group);
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_add_event_unmatched_group);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+  it('should apply delta mutations on added files from push event', async () => {
+    const config = new ConfigReader(mock.config_single_integration_branch);
+
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_add_event);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'delta',
+      added: mock.expected_added_location_entities,
+      removed: [],
+    });
+  });
+
+  it('should apply delta mutations on removed files from push event', async () => {
+    const config = new ConfigReader(mock.config_single_integration_branch);
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_remove_event);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'delta',
+      added: [],
+      removed: mock.expected_removed_location_entities,
+    });
+  });
+
+  it('should call refresh on added files from push event', async () => {
+    const config = new ConfigReader(mock.config_single_integration_branch);
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    const url = `https://example.com/group1/test-repo1`;
+
+    await events.publish(mock.push_modif_event);
+
+    expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.refresh).toHaveBeenCalledWith({
+      keys: [
+        `url:${url}/-/tree/main/catalog-info.yaml`,
+        `url:${url}/-/blob/main/catalog-info.yaml`,
+      ],
+    });
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+  // EventSupportChange: stop add tests >>>
+});

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts

@@ -17,12 +17,15 @@
 import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
 import { Config } from '@backstage/config';
 import { GitLabIntegration, ScmIntegrations } from '@backstage/integration';
+import { LocationSpec } from '@backstage/plugin-catalog-common';
 import {
+  DeferredEntity,
   EntityProvider,
   EntityProviderConnection,
-  LocationSpec,
   locationSpecToLocationEntity,
 } from '@backstage/plugin-catalog-node';
+import { EventsService } from '@backstage/plugin-events-node';
+import { WebhookProjectSchema, WebhookPushEventSchema } from '@gitbeaker/rest';
 import * as uuid from 'uuid';
 import {
   GitLabClient,
@@ -33,26 +36,38 @@ import {
 } from '../lib';
 import { LoggerService } from '@backstage/backend-plugin-api';
 
+import * as path from 'path';
+
+const TOPIC_REPO_PUSH = 'gitlab.push';
+
 type Result = {
   scanned: number;
   matches: GitLabProject[];
 };
 
 /**
- * Discovers entity definition files in the groups of a Gitlab instance.
+ * Discovers catalog files located in your GitLab instance.
+ * The provider will search your GitLab instance's projects and register catalog files matching the configured path
+ * as Location entity and via following processing steps add all contained catalog entities.
+ * This can be useful as an alternative to static locations or manually adding things to the catalog.
+ *
  * @public
  */
+// <<< EventSupportChange: implemented EventSubscriber interface
 export class GitlabDiscoveryEntityProvider implements EntityProvider {
   private readonly config: GitlabProviderConfig;
   private readonly integration: GitLabIntegration;
   private readonly logger: LoggerService;
   private readonly scheduleFn: () => Promise<void>;
   private connection?: EntityProviderConnection;
+  private readonly events?: EventsService;
+  private readonly gitLabClient: GitLabClient;
 
   static fromConfig(
     config: Config,
     options: {
       logger: LoggerService;
+      events?: EventsService;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
     },
@@ -92,13 +107,20 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
         }),
       );
     });
+
     return providers;
   }
 
+  /**
+   * Constructs a GitlabDiscoveryEntityProvider instance.
+   *
+   * @param options - Configuration options including config, integration, logger, and taskRunner.
+   */
   private constructor(options: {
     config: GitlabProviderConfig;
     integration: GitLabIntegration;
     logger: LoggerService;
+    events?: EventsService;
     taskRunner: TaskRunner;
   }) {
     this.config = options.config;
@@ -107,6 +129,11 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       target: this.getProviderName(),
     });
     this.scheduleFn = this.createScheduleFn(options.taskRunner);
+    this.events = options.events;
+    this.gitLabClient = new GitLabClient({
+      config: this.integration.config,
+      logger: this.logger,
+    });
   }
 
   getProviderName(): string {
@@ -116,8 +143,28 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
   async connect(connection: EntityProviderConnection): Promise<void> {
     this.connection = connection;
     await this.scheduleFn();
+
+    if (this.events) {
+      await this.events.subscribe({
+        id: this.getProviderName(),
+        topics: [TOPIC_REPO_PUSH],
+        onEvent: async params => {
+          if (params.topic !== TOPIC_REPO_PUSH) {
+            return;
+          }
+
+          await this.onRepoPush(params.eventPayload as WebhookPushEventSchema);
+        },
+      });
+    }
   }
 
+  /**
+   * Creates a scheduled task runner for refreshing the entity provider.
+   *
+   * @param taskRunner - The task runner instance.
+   * @returns The scheduled function.
+   */
   private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
     return async () => {
       const taskId = `${this.getProviderName()}:refresh`;
@@ -143,6 +190,11 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
     };
   }
 
+  /**
+   * Performs a full scan on the GitLab instance searching for locations to be ingested
+   *
+   * @param logger - The logger instance for logging.
+   */
   async refresh(logger: LoggerService): Promise<void> {
     if (!this.connection) {
       throw new Error(
@@ -150,13 +202,8 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       );
     }
 
-    const client = new GitLabClient({
-      config: this.integration.config,
-      logger: logger,
-    });
-
     const projects = paginated<GitLabProject>(
-      options => client.listProjects(options),
+      options => this.gitLabClient.listProjects(options),
       {
         archived: false,
         group: this.config.group,
@@ -171,43 +218,17 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
     };
 
     for await (const project of projects) {
-      if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) {
-        continue;
-      }
-
-      res.scanned++;
-
-      if (
-        this.config.skipForkedRepos &&
-        project.hasOwnProperty('forked_from_project')
-      ) {
-        continue;
-      }
-
-      if (
-        !this.config.branch &&
-        this.config.fallbackBranch === '*' &&
-        project.default_branch === undefined
-      ) {
-        continue;
-      }
-
-      const project_branch =
-        this.config.branch ??
-        project.default_branch ??
-        this.config.fallbackBranch;
-
-      const projectHasFile: boolean = await client.hasFile(
-        project.path_with_namespace ?? '',
-        project_branch,
-        this.config.catalogFile,
-      );
-      if (projectHasFile) {
+      if (await this.shouldProcessProject(project, this.gitLabClient)) {
+        res.scanned++;
         res.matches.push(project);
       }
     }
 
     const locations = res.matches.map(p => this.createLocationSpec(p));
+
+    logger.info(
+      `Processed ${locations.length} from scanned ${res.scanned} projects.`,
+    );
     await this.connection.applyMutation({
       type: 'full',
       entities: locations.map(location => ({
@@ -222,10 +243,250 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       this.config.branch ??
       project.default_branch ??
       this.config.fallbackBranch;
+
     return {
       type: 'url',
       target: `${project.web_url}/-/blob/${project_branch}/${this.config.catalogFile}`,
       presence: 'optional',
     };
   }
+
+  /**
+   * Handles the "gitlab.push" event.
+   *
+   * @param event - The push event payload.
+   */
+  private async onRepoPush(event: WebhookPushEventSchema): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+    this.logger.info(
+      `Received push event for ${event.project.path_with_namespace}`,
+    );
+
+    const project = await this.gitLabClient.getProjectById(event.project_id);
+
+    if (!project) {
+      this.logger.debug(
+        `Ignoring push event for ${event.project.path_with_namespace}`,
+      );
+
+      return;
+    }
+
+    if (!(await this.shouldProcessProject(project, this.gitLabClient))) {
+      this.logger.debug(`Skipping event ${event.project.path_with_namespace}`);
+      return;
+    }
+
+    // Get array of added, removed or modified files from the push event
+    const added = this.getFilesMatchingConfig(
+      event,
+      'added',
+      this.config.catalogFile,
+    );
+    const removed = this.getFilesMatchingConfig(
+      event,
+      'removed',
+      this.config.catalogFile,
+    );
+    const modified = this.getFilesMatchingConfig(
+      event,
+      'modified',
+      this.config.catalogFile,
+    );
+
+    // Modified files will be scheduled to a refresh
+    const addedEntities = this.createLocationSpecCommitedFiles(
+      event.project,
+      added,
+    );
+
+    const removedEntities = this.createLocationSpecCommitedFiles(
+      event.project,
+      removed,
+    );
+
+    if (addedEntities.length > 0 || removedEntities.length > 0) {
+      await this.connection.applyMutation({
+        type: 'delta',
+        added: this.toDeferredEntities(
+          addedEntities.map(entity => entity.target),
+        ),
+        removed: this.toDeferredEntities(
+          removedEntities.map(entity => entity.target),
+        ),
+      });
+    }
+
+    if (modified.length > 0) {
+      const projectBranch =
+        this.config.branch ??
+        event.project.default_branch ??
+        this.config.fallbackBranch;
+
+      // scheduling a refresh to both tree and blob (https://git-scm.com/book/en/v2/Git-Internals-Git-Objects)
+      await this.connection.refresh({
+        keys: [
+          ...modified.map(
+            filePath =>
+              `url:${event.project.web_url}/-/tree/${projectBranch}/${filePath}`,
+          ),
+          ...modified.map(
+            filePath =>
+              `url:${event.project.web_url}/-/blob/${projectBranch}/${filePath}`,
+          ),
+        ],
+      });
+    }
+
+    this.logger.info(
+      `Processed GitLab push event from ${event.project.web_url}: added ${added.length} - removed ${removed.length} - modified ${modified.length}`,
+    );
+  }
+
+  /**
+   * Gets files matching the specified commit action and catalog file name.
+   *
+   * @param event - The push event payload.
+   * @param action - The action type ('added', 'removed', or 'modified').
+   * @param catalogFile - The catalog file name.
+   * @returns An array of file paths.
+   */
+  private getFilesMatchingConfig(
+    event: WebhookPushEventSchema,
+    action: 'added' | 'removed' | 'modified',
+    catalogFile: string,
+  ): string[] {
+    if (!event.commits) {
+      return [];
+    }
+
+    const matchingFiles = event.commits.flatMap((element: any) =>
+      element[action].filter(
+        (file: string) => path.basename(file) === catalogFile,
+      ),
+    );
+
+    if (matchingFiles.length === 0) {
+      this.logger.debug(
+        `No files matching '${catalogFile}' found in the commits.`,
+      );
+    }
+
+    return matchingFiles;
+  }
+
+  /**
+   * Creates Backstage location specs for committed files.
+   *
+   * @param project - The GitLab project information.
+   * @param addedFiles - The array of added file paths.
+   * @returns An array of location specs.
+   */
+  private createLocationSpecCommitedFiles(
+    project: WebhookProjectSchema,
+    addedFiles: string[],
+  ): LocationSpec[] {
+    const projectBranch =
+      this.config.branch ??
+      project.default_branch ??
+      this.config.fallbackBranch;
+
+    // Filter added files that match the catalog file pattern
+    const matchingFiles = addedFiles.filter(
+      file => path.basename(file) === this.config.catalogFile,
+    );
+
+    // Create a location spec for each matching file
+    const locationSpecs: LocationSpec[] = matchingFiles.map(file => ({
+      type: 'url',
+      target: `${project.web_url}/-/blob/${projectBranch}/${file}`,
+      presence: 'optional',
+    }));
+
+    return locationSpecs;
+  }
+
+  /**
+   * Converts a target URL to a LocationSpec object.
+   *
+   * @param {string} target - The target URL to be converted.
+   * @returns {LocationSpec} The LocationSpec object representing the URL.
+   */
+  private toLocationSpec(target: string): LocationSpec {
+    return {
+      type: 'url',
+      target: target,
+      presence: 'optional',
+    };
+  }
+
+  private toDeferredEntities(targets: string[]): DeferredEntity[] {
+    return targets
+      .map(target => {
+        const location = this.toLocationSpec(target);
+
+        return locationSpecToLocationEntity({ location });
+      })
+      .map(entity => {
+        return {
+          locationKey: this.getProviderName(),
+          entity: entity,
+        };
+      });
+  }
+
+  private async shouldProcessProject(
+    project: GitLabProject,
+    client: GitLabClient,
+  ): Promise<boolean> {
+    if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) {
+      this.logger.debug(
+        `Skipping project ${project.path_with_namespace} as it does not match the project pattern ${this.config.projectPattern}.`,
+      );
+      return false;
+    }
+
+    if (
+      this.config.group &&
+      !project.path_with_namespace!.startsWith(`${this.config.group}/`)
+    ) {
+      this.logger.debug(
+        `Skipping project ${project.path_with_namespace} as it does not match the group pattern ${this.config.group}.`,
+      );
+      return false;
+    }
+
+    if (
+      this.config.skipForkedRepos &&
+      project.hasOwnProperty('forked_from_project')
+    ) {
+      this.logger.debug(
+        `Skipping project ${project.path_with_namespace} as it is a forked project.`,
+      );
+      return false;
+    }
+
+    const customFallbackBranch =
+      this.config.fallbackBranch !== 'master'
+        ? this.config.fallbackBranch
+        : undefined;
+
+    const project_branch =
+      this.config.branch ??
+      customFallbackBranch ??
+      project.default_branch ??
+      this.config.fallbackBranch;
+
+    const hasFile = await client.hasFile(
+      project.path_with_namespace ?? '',
+      project_branch,
+      this.config.catalogFile,
+    );
+
+    return hasFile;
+  }
 }

plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts

@@ -25,6 +25,7 @@ import {
   EntityProvider,
   EntityProviderConnection,
 } from '@backstage/plugin-catalog-node';
+import { EventsService } from '@backstage/plugin-events-node';
 import { merge } from 'lodash';
 import * as uuid from 'uuid';
 import { LoggerService } from '@backstage/backend-plugin-api';
@@ -35,21 +36,24 @@ import {
   paginated,
   readGitlabConfigs,
 } from '../lib';
-import {
-  GitLabGroup,
-  GitLabUser,
-  GroupNameTransformer,
-  GroupTransformer as GroupEntitiesTransformer,
-  PagedResponse,
-  UserTransformer,
-} from '../lib/types';
 import {
   defaultGroupEntitiesTransformer,
   defaultGroupNameTransformer,
   defaultUserTransformer,
 } from '../lib/defaultTransformers';
+import {
+  GitLabGroup,
+  GitLabUser,
+  GroupTransformer as GroupEntitiesTransformer,
+  GroupNameTransformer,
+  PagedResponse,
+  SystemHookBaseGroupEventsSchema,
+  SystemHookBaseMembershipEventsSchema,
+  SystemHookBaseUserEventsSchema,
+  UserTransformer,
+} from '../lib/types';
 
-type Result = {
+type UserResult = {
   scanned: number;
   matches: GitLabUser[];
 };
@@ -59,6 +63,36 @@ type GroupResult = {
   matches: GitLabGroup[];
 };
 
+type SystemHookGroupCreateOrDestroyEventSchema =
+  SystemHookBaseGroupEventsSchema & {
+    event_name: 'group_create' | 'group_destroy';
+  };
+
+type SystemHookGroupRenameEventSchema = SystemHookBaseGroupEventsSchema & {
+  event_name: 'group_rename';
+  old_path: string;
+  old_full_path: string;
+};
+
+type SystemHookUserCreateOrDestroyEventSchema =
+  SystemHookBaseUserEventsSchema & {
+    event_name: 'user_create' | 'user_destroy';
+  };
+
+type SystemHookCreateOrDestroyMembershipEventsSchema =
+  SystemHookBaseMembershipEventsSchema & {
+    event_name: 'user_add_to_group' | 'user_remove_from_group';
+  };
+
+// System level events
+const TOPIC_GROUP_CREATE = 'gitlab.group_create';
+const TOPIC_GROUP_DESTROY = 'gitlab.group_destroy';
+const TOPIC_GROUP_RENAME = 'gitlab.group_rename';
+const TOPIC_USER_CREATE = 'gitlab.user_create';
+const TOPIC_USER_DESTROY = 'gitlab.user_destroy';
+const TOPIC_USER_ADD_GROUP = 'gitlab.user_add_to_group';
+const TOPIC_USER_REMOVE_GROUP = 'gitlab.user_remove_from_group';
+
 /**
  * Discovers users and groups from a Gitlab instance.
  * @public
@@ -67,16 +101,19 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
   private readonly config: GitlabProviderConfig;
   private readonly integration: GitLabIntegration;
   private readonly logger: LoggerService;
+  private readonly events?: EventsService;
   private readonly scheduleFn: () => Promise<void>;
   private connection?: EntityProviderConnection;
   private userTransformer: UserTransformer;
   private groupEntitiesTransformer: GroupEntitiesTransformer;
   private groupNameTransformer: GroupNameTransformer;
+  private readonly gitLabClient: GitLabClient;
 
   static fromConfig(
     config: Config,
     options: {
       logger: LoggerService;
+      events?: EventsService;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
       userTransformer?: UserTransformer;
@@ -96,7 +133,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       const integration = integrations.byHost(providerConfig.host);
 
       if (!providerConfig.orgEnabled) {
-        return;
+        throw new Error(`Org not enabled for ${providerConfig.id}.`);
       }
 
       if (!integration) {
@@ -137,6 +174,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     config: GitlabProviderConfig;
     integration: GitLabIntegration;
     logger: LoggerService;
+    events?: EventsService;
     taskRunner: TaskRunner;
     userTransformer?: UserTransformer;
     groupEntitiesTransformer?: GroupEntitiesTransformer;
@@ -148,11 +186,18 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       target: this.getProviderName(),
     });
     this.scheduleFn = this.createScheduleFn(options.taskRunner);
+    this.events = options.events;
+
     this.userTransformer = options.userTransformer ?? defaultUserTransformer;
     this.groupEntitiesTransformer =
       options.groupEntitiesTransformer ?? defaultGroupEntitiesTransformer;
     this.groupNameTransformer =
       options.groupNameTransformer ?? defaultGroupNameTransformer;
+
+    this.gitLabClient = new GitLabClient({
+      config: this.integration.config,
+      logger: this.logger,
+    });
   }
 
   getProviderName(): string {
@@ -162,6 +207,117 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
   async connect(connection: EntityProviderConnection): Promise<void> {
     this.connection = connection;
     await this.scheduleFn();
+
+    // Specifies which topics will be listened to.
+    // The topics from the original GitLab events contain only the string 'gitlab'. These are caught by the GitlabEventRouter Module, who republishes them with a more specific topic 'gitlab.<event_name>'
+    if (this.events) {
+      await this.events.subscribe({
+        id: this.getProviderName(),
+        topics: [
+          TOPIC_GROUP_CREATE,
+          TOPIC_GROUP_DESTROY,
+          TOPIC_GROUP_RENAME,
+          TOPIC_USER_CREATE,
+          TOPIC_USER_DESTROY,
+          TOPIC_USER_ADD_GROUP,
+          TOPIC_USER_REMOVE_GROUP,
+        ],
+        onEvent: async params => {
+          this.logger.info(`Received event from topic ${params.topic}`);
+
+          const addEntitiesOperation = (entities: Entity[]) => ({
+            removed: [],
+            added: entities.map(entity => ({
+              locationKey: this.getProviderName(),
+              entity: this.withLocations(
+                this.integration.config.host,
+                this.integration.config.baseUrl,
+                entity,
+              ),
+            })),
+          });
+
+          const removeEntitiesOperation = (entities: Entity[]) => ({
+            added: [],
+            removed: entities.map(entity => ({
+              locationKey: this.getProviderName(),
+              entity: this.withLocations(
+                this.integration.config.host,
+                this.integration.config.baseUrl,
+                entity,
+              ),
+            })),
+          });
+
+          const replaceEntitiesOperation = (entities: Entity[]) => {
+            const entitiesToReplace = entities.map(entity => ({
+              locationKey: this.getProviderName(),
+              entity: this.withLocations(
+                this.integration.config.host,
+                this.integration.config.baseUrl,
+                entity,
+              ),
+            }));
+
+            return {
+              removed: entitiesToReplace,
+              added: entitiesToReplace,
+            };
+          };
+
+          // handle group change events
+          if (
+            params.topic === TOPIC_GROUP_CREATE ||
+            params.topic === TOPIC_GROUP_DESTROY
+          ) {
+            const payload: SystemHookGroupCreateOrDestroyEventSchema =
+              params.eventPayload as SystemHookGroupCreateOrDestroyEventSchema;
+
+            const createDeltaOperation =
+              params.topic === TOPIC_GROUP_CREATE
+                ? addEntitiesOperation
+                : removeEntitiesOperation;
+
+            await this.onGroupChange(payload, createDeltaOperation);
+          }
+          if (params.topic === TOPIC_GROUP_RENAME) {
+            const payload: SystemHookGroupRenameEventSchema =
+              params.eventPayload as SystemHookGroupRenameEventSchema;
+
+            await this.onGroupEdit(payload, replaceEntitiesOperation);
+          }
+
+          // handle user change events
+          if (
+            params.topic === TOPIC_USER_CREATE ||
+            params.topic === TOPIC_USER_DESTROY
+          ) {
+            const payload: SystemHookUserCreateOrDestroyEventSchema =
+              params.eventPayload as SystemHookUserCreateOrDestroyEventSchema;
+
+            const createDeltaOperation =
+              params.topic === TOPIC_USER_CREATE
+                ? addEntitiesOperation
+                : removeEntitiesOperation;
+
+            await this.onUserChange(payload, createDeltaOperation);
+          }
+
+          // handle user membership changes
+          if (
+            params.topic === TOPIC_USER_ADD_GROUP ||
+            params.topic === TOPIC_USER_REMOVE_GROUP
+          ) {
+            const payload: SystemHookCreateOrDestroyMembershipEventsSchema =
+              params.eventPayload as SystemHookCreateOrDestroyMembershipEventsSchema;
+
+            const createDeltaOperation = addEntitiesOperation;
+
+            await this.onMembershipChange(payload, createDeltaOperation);
+          }
+        },
+      });
+    }
   }
 
   private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
@@ -196,30 +352,33 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       );
     }
 
-    const client = new GitLabClient({
-      config: this.integration.config,
-      logger: logger,
-    });
-
     let groups;
     let users;
 
-    if (client.isSelfManaged()) {
-      groups = paginated<GitLabGroup>(options => client.listGroups(options), {
-        page: 1,
-        per_page: 100,
-      });
+    if (this.gitLabClient.isSelfManaged()) {
+      groups = paginated<GitLabGroup>(
+        options => this.gitLabClient.listGroups(options),
+        {
+          page: 1,
+          per_page: 100,
+          all_available: true,
+        },
+      );
 
-      users = paginated<GitLabUser>(options => client.listUsers(options), {
-        page: 1,
-        per_page: 100,
-        active: true,
-      });
+      users = paginated<GitLabUser>(
+        options => this.gitLabClient.listUsers(options),
+        {
+          page: 1,
+          per_page: 100,
+          active: true,
+        },
+      );
     } else {
-      groups = (await client.listDescendantGroups(this.config.group)).items;
+      groups = (await this.gitLabClient.listDescendantGroups(this.config.group))
+        .items;
       const rootGroup = this.config.group.split('/')[0];
       users = paginated<GitLabUser>(
-        options => client.listSaaSUsers(rootGroup, options),
+        options => this.gitLabClient.listSaaSUsers(rootGroup, options),
         {
           page: 1,
           per_page: 100,
@@ -229,7 +388,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
 
     const idMappedUser: { [userId: number]: GitLabUser } = {};
 
-    const res: Result = {
+    const userRes: UserResult = {
       scanned: 0,
       matches: [],
     };
@@ -240,46 +399,45 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     };
 
     for await (const user of users) {
-      if (!this.config.userPattern.test(user.email ?? user.username ?? '')) {
-        continue;
-      }
+      userRes.scanned++;
 
-      res.scanned++;
-
-      if (user.state !== 'active') {
+      if (!this.shouldProcessUser(user)) {
+        logger.debug(`Skipped user: ${user.username}`);
         continue;
       }
 
       idMappedUser[user.id] = user;
-      res.matches.push(user);
+      userRes.matches.push(user);
     }
 
     for await (const group of groups) {
-      if (!this.config.groupPattern.test(group.full_path ?? '')) {
-        continue;
-      }
-
-      if (
-        this.config.group &&
-        !group.full_path.startsWith(`${this.config.group}/`)
-      ) {
-        continue;
-      }
-
       groupRes.scanned++;
+
+      if (!this.shouldProcessGroup(group)) {
+        logger.debug(`Skipped group: ${group.full_path}`);
+        continue;
+      }
+      logger.debug(`Processed group: ${group.full_path}`);
+
       groupRes.matches.push(group);
 
       let groupUsers: PagedResponse<GitLabUser> = { items: [] };
       try {
-        groupUsers = await client.getGroupMembers(group.full_path, ['DIRECT']);
+        const relations = this.config.allowInherited
+          ? ['DIRECT', 'INHERITED']
+          : ['DIRECT'];
+        groupUsers = await this.gitLabClient.getGroupMembers(
+          group.full_path,
+          relations,
+        );
       } catch (e) {
         logger.error(
           `Failed fetching users for group '${group.full_path}': ${e}`,
         );
       }
-
       for (const groupUser of groupUsers.items) {
         const user = idMappedUser[groupUser.id];
+
         if (user) {
           user.groups = (user.groups ?? []).concat(group);
         }
@@ -288,13 +446,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
 
     const groupsWithUsers = groupRes.matches.filter(group => {
       return (
-        res.matches.filter(x => {
+        userRes.matches.filter(x => {
           return !!x.groups?.find(y => y.id === group.id);
         }).length > 0
       );
     });
 
-    const userEntities = res.matches.map(p =>
+    const userEntities = userRes.matches.map(p =>
       this.userTransformer({
         user: p,
         integrationConfig: this.integration.config,
@@ -309,6 +467,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       groupNameTransformer: this.groupNameTransformer,
     });
 
+    logger.info(
+      `Scanned ${userRes.scanned} users and processed ${userEntities.length} users`,
+    );
+    logger.info(
+      `Scanned ${groupRes.scanned} groups and processed ${groupEntities.length} groups`,
+    );
+
     await this.connection.applyMutation({
       type: 'full',
       entities: [...userEntities, ...groupEntities].map(entity => ({
@@ -321,6 +486,282 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       })),
     });
   }
+  private async onGroupChange(
+    event: SystemHookGroupCreateOrDestroyEventSchema,
+    createDeltaOperation: (entities: Entity[]) => {
+      added: any[];
+      removed: any[];
+    },
+  ): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+
+    let group: GitLabGroup | undefined;
+    if (event.event_name === 'group_destroy') {
+      group = {
+        id: event.group_id,
+        full_path: event.full_path,
+        name: event.name,
+        description: '',
+        parent_id: 0,
+      };
+    } else {
+      group = await this.gitLabClient.getGroupById(event.group_id);
+    }
+
+    if (!this.shouldProcessGroup(group)) {
+      this.logger.debug(`Skipped group ${group.full_path}.`);
+      return;
+    }
+
+    // create the group entity
+    const groupEntity = this.groupEntitiesTransformer({
+      groups: [group],
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+
+    // we need to fetch the parent group's object because its representation might be changed by the groupTransformer
+    if (group.parent_id) {
+      const parentGroup = await this.gitLabClient.getGroupById(group.parent_id);
+
+      groupEntity[0].spec.parent = this.groupNameTransformer({
+        group: parentGroup,
+        providerConfig: this.config,
+      });
+    }
+
+    this.logger.debug(`Applying mutation for group ${group.full_path}.`);
+    await this.connection.applyMutation({
+      type: 'delta',
+      ...createDeltaOperation(groupEntity),
+    });
+  }
+
+  // the goal here is to trigger a mutation to remove the old entity and add the new one.
+  private async onGroupEdit(
+    event: SystemHookGroupRenameEventSchema,
+    createDeltaOperation: (entities: Entity[]) => {
+      added: any[];
+      removed: any[];
+    },
+  ): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+
+    const groupToRemove: GitLabGroup = {
+      id: event.group_id,
+      full_path: event.old_full_path,
+      name: event.name,
+      description: '',
+      parent_id: 0,
+    };
+
+    if (!this.shouldProcessGroup(groupToRemove)) {
+      this.logger.debug(`Skipped group ${groupToRemove.full_path}.`);
+      return;
+    }
+
+    const groupEntityToRemove = await this.groupEntitiesTransformer({
+      groups: [groupToRemove],
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+
+    const groupToAdd = await this.gitLabClient.getGroupById(event.group_id);
+
+    if (!this.shouldProcessGroup(groupToAdd)) {
+      this.logger.debug(`Skipped group ${groupToAdd.full_path}.`);
+      return;
+    }
+
+    const groupEntityToAdd = await this.groupEntitiesTransformer({
+      groups: [groupToAdd],
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+
+    if (groupToAdd.parent_id) {
+      const parentGroup = await this.gitLabClient.getGroupById(
+        groupToAdd.parent_id,
+      );
+
+      groupEntityToAdd[0].spec.parent = this.groupNameTransformer({
+        group: parentGroup,
+        providerConfig: this.config,
+      });
+    }
+
+    const { added } = createDeltaOperation(groupEntityToAdd);
+    const { removed } = createDeltaOperation(groupEntityToRemove);
+
+    this.logger.debug(`Applying mutation for group ${groupToAdd.full_path}.`);
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  private async onUserChange(
+    event: SystemHookUserCreateOrDestroyEventSchema,
+    createDeltaOperation: (entities: Entity[]) => {
+      added: any[];
+      removed: any[];
+    },
+  ): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+
+    let user: GitLabUser | undefined = undefined;
+
+    // if user destroy event is received, retrieve user data from the event itself
+    if (event.event_name === 'user_destroy') {
+      user = {
+        id: event.user_id,
+        username: event.username,
+        email: event.email,
+        name: event.name,
+        state: 'active', // in the delete case it doesn't really matter if the user is active or not
+        web_url: '',
+        avatar_url: '',
+      };
+    }
+
+    // if user create event received fetch data from gitlab
+    if (event.event_name === 'user_create') {
+      user = await this.gitLabClient.getUserById(event.user_id);
+    }
+
+    if (!user) {
+      this.logger.debug(
+        `Couldn't retrieve user data. Skipped ${event.event_name} event processing for user ${event.username}`,
+      );
+      return;
+    }
+
+    if (!this.shouldProcessUser(user)) {
+      this.logger.debug(`Skipped user ${user.username}.`);
+      return;
+    }
+
+    const userEntity = await this.userTransformer({
+      user: user,
+      integrationConfig: this.integration.config,
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+    const { added, removed } = createDeltaOperation([userEntity]);
+
+    this.logger.debug(`Applying mutation for user ${user.username}.`);
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  // the goal here is to reconstruct the group either from which the user was removed or to which the user was added. Specifically, we add/remove the new user to/from the spec.member property array of the group entity. The Processor should take care of updating the relations
+  private async onMembershipChange(
+    event: SystemHookCreateOrDestroyMembershipEventsSchema,
+    createDeltaOperation: (entities: Entity[]) => {
+      added: any[];
+      removed: any[];
+    },
+  ): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+
+    // fetch group data from GitLab
+    const groupToRebuild: GitLabGroup = await this.gitLabClient.getGroupById(
+      event.group_id,
+    );
+
+    if (!groupToRebuild) {
+      this.logger.debug(
+        `Couldn't retrieve group data. Skipped ${event.event_name} event processing.`,
+      );
+      return;
+    }
+
+    // if the group is outside the scope there is no point creating anything related to it.
+    if (!this.shouldProcessGroup(groupToRebuild)) {
+      this.logger.debug(`Skipped group ${groupToRebuild.full_path}.`);
+      return;
+    }
+
+    const relations = this.config.allowInherited
+      ? ['DIRECT', 'INHERITED']
+      : ['DIRECT'];
+
+    // fetch group members from GitLab
+    const groupMembers = await this.gitLabClient.getGroupMembers(
+      groupToRebuild.full_path,
+      relations,
+    );
+    const usersToBeAdded = groupMembers.items;
+
+    const groupEntityToModify = await this.groupEntitiesTransformer({
+      groups: [groupToRebuild],
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+
+    // we need to fetch the parent group's object because its representation might be changed by the groupTransformer
+    if (groupToRebuild.parent_id) {
+      const parentGroup = await this.gitLabClient.getGroupById(
+        groupToRebuild.parent_id,
+      );
+
+      // update parent of the group entity
+      groupEntityToModify[0].spec.parent = this.groupNameTransformer({
+        group: parentGroup,
+        providerConfig: this.config,
+      });
+    }
+
+    // update members of the group entity
+    groupEntityToModify[0].spec.members =
+      usersToBeAdded.length !== 0 ? usersToBeAdded.map(e => e.username) : [];
+
+    const { added, removed } = createDeltaOperation(groupEntityToModify);
+
+    this.logger.debug(
+      `Applying mutation for group ${groupToRebuild.full_path}.`,
+    );
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  private shouldProcessGroup(group: GitLabGroup): boolean {
+    return (
+      this.config.groupPattern.test(group.full_path) &&
+      (!this.config.group ||
+        group.full_path.startsWith(`${this.config.group}/`))
+    );
+  }
+
+  private shouldProcessUser(user: GitLabUser): boolean {
+    return (
+      this.config.userPattern.test(user.email ?? user.username ?? '') &&
+      user.state === 'active'
+    );
+  }
 
   private withLocations(host: string, baseUrl: string, entity: Entity): Entity {
     const location =

plugins/catalog-backend-module-gitlab/src/providers/config.test.ts

@@ -58,6 +58,7 @@ describe('config', () => {
         groupPattern: /[\s\S]*/,
         userPattern: /[\s\S]*/,
         orgEnabled: false,
+        allowInherited: false,
         schedule: undefined,
         skipForkedRepos: false,
       }),
@@ -95,6 +96,7 @@ describe('config', () => {
         groupPattern: /[\s\S]*/,
         userPattern: /[\s\S]*/,
         orgEnabled: false,
+        allowInherited: false,
         schedule: undefined,
         skipForkedRepos: false,
       }),
@@ -133,6 +135,7 @@ describe('config', () => {
         groupPattern: /[\s\S]*/,
         userPattern: /[\s\S]*/,
         orgEnabled: false,
+        allowInherited: false,
         schedule: undefined,
         skipForkedRepos: true,
       }),
@@ -173,6 +176,7 @@ describe('config', () => {
         groupPattern: /[\s\S]*/,
         userPattern: /[\s\S]*/,
         orgEnabled: false,
+        allowInherited: false,
         skipForkedRepos: false,
         schedule: {
           frequency: Duration.fromISO('PT30M'),

plugins/catalog-backend-module-gitlab/src/providers/config.ts

@@ -43,6 +43,8 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
     config.getOptionalString('groupPattern') ?? /[\s\S]*/,
   );
   const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false;
+  const allowInherited: boolean =
+    config.getOptionalBoolean('allowInherited') ?? false;
   const skipForkedRepos: boolean =
     config.getOptionalBoolean('skipForkedRepos') ?? false;
 
@@ -62,6 +64,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
     groupPattern,
     schedule,
     orgEnabled,
+    allowInherited,
     skipForkedRepos,
   };
 }

yarn.lock

@@ -5013,7 +5013,24 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab":
+"@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org":
+  version: 0.0.0-use.local
+  resolution: "@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org"
+  dependencies:
+    "@backstage/backend-common": "workspace:^"
+    "@backstage/backend-plugin-api": "workspace:^"
+    "@backstage/backend-tasks": "workspace:^"
+    "@backstage/backend-test-utils": "workspace:^"
+    "@backstage/cli": "workspace:^"
+    "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^"
+    "@backstage/plugin-catalog-node": "workspace:^"
+    "@backstage/plugin-events-backend-test-utils": "workspace:^"
+    "@backstage/plugin-events-node": "workspace:^"
+    luxon: ^3.0.0
+  languageName: unknown
+  linkType: soft
+
+"@backstage/plugin-catalog-backend-module-gitlab@workspace:^, @backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab":
   version: 0.0.0-use.local
   resolution: "@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab"
   dependencies:
@@ -5025,7 +5042,11 @@ __metadata:
     "@backstage/cli": "workspace:^"
     "@backstage/config": "workspace:^"
     "@backstage/integration": "workspace:^"
+    "@backstage/plugin-catalog-common": "workspace:^"
     "@backstage/plugin-catalog-node": "workspace:^"
+    "@backstage/plugin-events-backend-test-utils": "workspace:^"
+    "@backstage/plugin-events-node": "workspace:^"
+    "@gitbeaker/rest": ^40.0.3
     "@types/lodash": ^4.14.151
     "@types/uuid": ^9.0.0
     lodash: ^4.17.21
@@ -8602,6 +8623,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@gitbeaker/core@npm:^40.0.3":
+  version: 40.0.3
+  resolution: "@gitbeaker/core@npm:40.0.3"
+  dependencies:
+    "@gitbeaker/requester-utils": ^40.0.3
+    qs: ^6.11.2
+    xcase: ^2.0.1
+  checksum: 999cea3f1523eabfae473e8a21849fdd4f6b4d30f0de681e735b238713b198ff9381c35d2ba5949d27ddc7ed5a953b14fc5a6ed38a5d98b05b6a5670a0c69162
+  languageName: node
+  linkType: hard
+
 "@gitbeaker/node@npm:^35.1.0, @gitbeaker/node@npm:^35.8.0":
   version: 35.8.1
   resolution: "@gitbeaker/node@npm:35.8.1"
@@ -8638,6 +8670,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@gitbeaker/requester-utils@npm:^40.0.3":
+  version: 40.0.3
+  resolution: "@gitbeaker/requester-utils@npm:40.0.3"
+  dependencies:
+    picomatch-browser: ^2.2.6
+    qs: ^6.11.2
+    rate-limiter-flexible: ^4.0.0
+    xcase: ^2.0.1
+  checksum: 1a46dc9c730c29e2229b04ed9bbe6cd87b7e5f16c1d0a5e78106f0287ab3d981ce2b099eb41cdb0f9180bb6b11d25f5f1fa92a91b69682bcd4ac986e58b55d31
+  languageName: node
+  linkType: hard
+
 "@gitbeaker/rest@npm:^39.25.0":
   version: 39.34.3
   resolution: "@gitbeaker/rest@npm:39.34.3"
@@ -8648,6 +8692,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@gitbeaker/rest@npm:^40.0.3":
+  version: 40.0.3
+  resolution: "@gitbeaker/rest@npm:40.0.3"
+  dependencies:
+    "@gitbeaker/core": ^40.0.3
+    "@gitbeaker/requester-utils": ^40.0.3
+  checksum: 6a6427b82ee3357fbe07e1235de7a3c98d50155898180fbe4dd641c8575faa93853f1a39185e18876f84469e1a385ccd9beb26fd95b61c260b81eb0ed342355b
+  languageName: node
+  linkType: hard
+
 "@google-cloud/container@npm:^5.0.0":
   version: 5.12.0
   resolution: "@google-cloud/container@npm:5.12.0"