Merge pull request #23373 from elaine-mattos/feat/add-event-support-gitlab-provider

Add event support for the Catalog Plugin GitLab Discovery Entity providers

docs/integrations/gitlab/discovery.md

@@ -11,28 +11,11 @@ entities from GitLab. The entity provider will crawl the GitLab instance and reg
 entities matching the configured paths. This can be useful as an alternative to
 static locations or manually adding things to the catalog.
 
-To use the discovery provider, you'll need a GitLab integration
-[set up](locations.md) with a `token`. Then you can add a provider config per group
-to the catalog configuration:
+This provider can also be configured to ingest GitLab data based on [GitLab Webhooks](https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#configure-a-webhook-in-gitlab). The events currently accepted are:
 
-```yaml
-catalog:
-  providers:
-    gitlab:
-      yourProviderId:
-        host: gitlab-host # Identifies one of the hosts set up in the integrations
-        branch: main # Optional. Used to discover on a specific branch
-        fallbackBranch: main # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default
-        skipForkedRepos: false # Optional. If the project is a fork, skip repository
-        group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned
-        entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
-        projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything
-        schedule: # optional; same options as in TaskScheduleDefinition
-          # supports cron, ISO duration, "human duration" as used in code
-          frequency: { minutes: 30 }
-          # supports ISO duration, "human duration" as used in code
-          timeout: { minutes: 3 }
-```
+- [`push`](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#push-events).
+
+## Installation
 
 As this provider is not one of the default providers, you will first need to install
 the gitlab catalog plugin:
@@ -42,15 +25,93 @@ the gitlab catalog plugin:
 yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab
 ```
 
-Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`:
+### Installation with New Backend System
+
+Then add the following to your backend initialization:
+
+```ts title="packages/backend/src/index.ts
+// optional if you want HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend/alpha'));
+// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha'));
+// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md
+// backend.add(eventsModuleGitlabEventRouter());
+// optional - token validator for the gitlab topic
+// backend.add(eventsModuleGitlabWebhook());
+backend.add(import('@backstage/plugin-catalog-backend-module-gitlab/alpha'));
+```
+
+You need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Further documentation:
+
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+### Installation with Legacy Backend System
+
+#### Installation without Events Support
+
+Add the segment below to `packages/backend/src/plugins/catalog.ts`:
 
 ```ts title="packages/backend/src/plugins/catalog.ts"
+/* highlight-add-next-line */
 import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
 
-const builder = await CatalogBuilder.create(env);
-/** ... other processors and/or providers ... */
-builder.addEntityProvider(
-  ...GitlabDiscoveryEntityProvider.fromConfig(env.config, {
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  /* highlight-add-start */
+  builder.addEntityProvider(
+    ...GitlabDiscoveryEntityProvider.fromConfig(env.config, {
+      logger: env.logger,
+      // optional: alternatively, use scheduler with schedule defined in app-config.yaml
+      schedule: env.scheduler.createScheduledTaskRunner({
+        frequency: { minutes: 30 },
+        timeout: { minutes: 3 },
+      }),
+      // optional: alternatively, use schedule
+      scheduler: env.scheduler,
+    }),
+  );
+  /* highlight-add-end */
+  // ..
+}
+```
+
+#### Installation with Events Support
+
+Please follow the installation instructions at
+
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+Additionally, you need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Set up your provider
+
+```ts title="packages/backend/src/plugins/catalog.ts"
+import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+/* highlight-add-next-line */
+import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
+import { Router } from 'express';
+import { PluginEnvironment } from '../types';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new ScaffolderEntitiesProcessor());
+  /* highlight-add-start */
+  const gitlabProvider = GitlabDiscoveryEntityProvider.fromConfig(env.config, {
     logger: env.logger,
     // optional: alternatively, use scheduler with schedule defined in app-config.yaml
     schedule: env.scheduler.createScheduledTaskRunner({
@@ -59,19 +120,45 @@ builder.addEntityProvider(
     }),
     // optional: alternatively, use schedule
     scheduler: env.scheduler,
-  }),
-);
+  });
+  env.eventBroker.subscribe(gitlabProvider);
+  builder.addEntityProvider(gitlabProvider);
+  /* highlight-add-end */
+  const { processingEngine, router } = await builder.build();
+  await processingEngine.start();
+  return router;
+}
+```
+
+## Configuration
+
+To use the discovery provider, you'll need a GitLab integration
+[set up](locations.md) with a `token`. Then you can add a provider config per group
+to the catalog configuration.
+
+> > NOTE: if you are using the New Backend System, the `schedule` has to be setup in the config, as shown below.
+
+```yaml title="app-config.yaml"
+catalog:
+  providers:
+    gitlab:
+      yourProviderId:
+        host: gitlab-host # Identifies one of the hosts set up in the integrations
+        branch: main # Optional. Used to discover on a specific branch
+        fallbackBranch: master # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default
+        skipForkedRepos: false # Optional. If the project is a fork, skip repository
+        group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned
+        entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
+        projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything
+        schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System
+          # supports cron, ISO duration, "human duration" as used in code
+          frequency: { minutes: 30 }
+          # supports ISO duration, "human duration" as used in code
+          timeout: { minutes: 3 }
 ```
 
 ## Alternative processor
 
-```yaml
-catalog:
-  locations:
-    - type: gitlab-discovery
-      target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml
-```
-
 As alternative to the entity provider `GitlabDiscoveryEntityProvider`
 you can still use the `GitLabDiscoveryProcessor`.
 
@@ -109,6 +196,15 @@ export default async function createPlugin(
 }
 ```
 
+And add the following to your app-config.yaml
+
+```yaml
+catalog:
+  locations:
+    - type: gitlab-discovery
+      target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml
+```
+
 If you don't want create location object if file with component definition do not exists in project, you can set the `skipReposWithoutExactFileMatch` option. That can reduce count of request to gitlab with 404 status code.
 
 If you don't want to create location object if the project is a fork, you can set the `skipForkedRepos` option.

docs/integrations/gitlab/org.md

@@ -11,34 +11,134 @@ groups -- directly from GitLab. The result is a hierarchy of
 [`Group`](../../features/software-catalog/descriptor-format.md#kind-group)
 entities that mirrors your org setup.
 
+This provider can also be configured to ingest GitLab data based on [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html). The events currently accepted are:
+
+- `group_create`
+- `group_destroy`
+- `group_rename`
+- `user_create`
+- `user_destroy`
+- `user_add_to_group`
+- `user_remove_from_group`
+
+## Installation
+
 As this provider is not one of the default providers, you will first need to install the Gitlab provider plugin:
 
 ```bash
 # From your Backstage root directory
-yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab
+yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab @backstage/plugin-catalog-backend-module-gitlab-org
 ```
 
-Then add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`:
+### Installation with New Backend System
+
+Then add the following to your backend initialization:
+
+```ts title="packages/backend/src/index.ts
+// optional if you want HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend/alpha'));
+// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha'));
+// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md
+// backend.add(eventsModuleGitlabEventRouter());
+// optional - token validator for the gitlab topic
+// backend.add(eventsModuleGitlabWebhook());
+backend.add(import('@backstage/plugin-catalog-backend-module-gitlab-org'));
+```
+
+You need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Further documentation:
+
+- [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html)
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+### Installation with Legacy Backend System
+
+#### Installation without Events Support
+
+Add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`:
 
 ```ts
 /* packages/backend/src/plugins/catalog.ts */
 /* highlight-add-next-line */
 import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
 
-const builder = await CatalogBuilder.create(env);
-/** ... other processors and/or providers ... */
-/* highlight-add-start */
-builder.addEntityProvider(
-  ...GitlabOrgDiscoveryEntityProvider.fromConfig(env.config, {
-    logger: env.logger,
-    // optional: alternatively, use scheduler with schedule defined in app-config.yaml
-    schedule: env.scheduler.createScheduledTaskRunner({
-      frequency: { minutes: 30 },
-      timeout: { minutes: 3 },
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  /** ... other processors and/or providers ... */
+  /* highlight-add-start */
+  builder.addEntityProvider(
+    ...GitlabOrgDiscoveryEntityProvider.fromConfig(env.config, {
+      logger: env.logger,
+      // optional: alternatively, use scheduler with schedule defined in app-config.yaml
+      schedule: env.scheduler.createScheduledTaskRunner({
+        frequency: { minutes: 30 },
+        timeout: { minutes: 3 },
+      }),
+      // optional: alternatively, use schedule
+      scheduler: env.scheduler,
     }),
-  }),
-);
-/* highlight-add-end */
+  );
+  /* highlight-add-end */
+  // ..
+}
+```
+
+#### Installation with Events Support
+
+Please follow the installation instructions at
+
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+Additionally, you need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Set up your provider
+
+```ts title="packages/backend/src/plugins/catalog.ts"
+import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+/* highlight-add-next-line */
+import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
+import { Router } from 'express';
+import { PluginEnvironment } from '../types';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new ScaffolderEntitiesProcessor());
+  /* highlight-add-start */
+  const gitlabOrgProvider = GitlabOrgDiscoveryEntityProvider.fromConfig(
+    env.config,
+    {
+      logger: env.logger,
+      // optional: alternatively, use scheduler with schedule defined in app-config.yaml
+      schedule: env.scheduler.createScheduledTaskRunner({
+        frequency: { minutes: 30 },
+        timeout: { minutes: 3 },
+      }),
+      // optional: alternatively, use schedule
+      scheduler: env.scheduler,
+    },
+  );
+  env.eventBroker.subscribe(gitlabOrgProvider);
+  builder.addEntityProvider(gitlabOrgProvider);
+  /* highlight-add-end */
+  const { processingEngine, router } = await builder.build();
+  await processingEngine.start();
+  return router;
+}
 ```
 
 ## Configuration
@@ -58,6 +158,8 @@ amount of data, this can take significant time and resources.
 The token used must have the `read_api` scope, and the Users and Groups fetched
 will be those visible to the account which provisioned the token.
 
+> > NOTE: if you are using the New Backend System, the `schedule` has to be setup in the config, as shown below.
+
 ```yaml
 catalog:
   providers:
@@ -66,7 +168,13 @@ catalog:
         host: gitlab.com
         orgEnabled: true
         group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped)
+        allowInherited: true # Allow groups to be ingested even if there are no direct members.
         groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything
+        schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System.
+          # supports cron, ISO duration, "human duration" as used in code
+          frequency: { minutes: 30 }
+          # supports ISO duration, "human duration" as used in code
+          timeout: { minutes: 3 }
 ```
 
 ### Groups
@@ -107,6 +215,74 @@ catalog:
           - allow: [Group, User]
 ```
 
+### Custom Transformers
+
+You can inject your own transformation logic to help map GitLab API responses
+into Backstage entities. You can do this on the user and group requests to
+enable you to do further processing or updates to these entities.
+
+To enable this you pass a function into the `GitlabOrgDiscoveryEntityProvider`. You can
+pass a `UserTransformer`, a `GroupEntitiesTransformer` or a `GroupNameTransformer` (or all of them)). The function is invoked
+for each item (user or group) that is returned from the API.
+
+The example below uses the groupNameTransformer option to change the metadata.name property of the Backstage Group Entity. Instead of populating it with the usual `group.full_path` data that comes from GitLab, it uses the `group.id`:
+
+```ts
+import { loggerToWinstonLogger } from '@backstage/backend-common';
+import {
+  coreServices,
+  createBackendModule,
+} from '@backstage/backend-plugin-api';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
+import {
+  GitlabOrgDiscoveryEntityProvider,
+  /* highlight-add-next-line */
+  GroupNameTransformerOptions,
+} from '@backstage/plugin-catalog-backend-module-gitlab';
+
+/* highlight-add-start */
+function customGroupNameTransformer(
+  options: GroupNameTransformerOptions,
+): string {
+  return `${options.group.id}`;
+}
+/* highlight-add-end */
+
+/**
+ * Registers the GitlabDiscoveryEntityProvider with the catalog processing extension point.
+ *
+ * @alpha
+ */
+export const catalogModuleGitlabOrgDiscoveryEntityProvider =
+  createBackendModule({
+    pluginId: 'catalog',
+    moduleId: 'gitlabOrgDiscoveryEntityProvider',
+    register(env) {
+      env.registerInit({
+        deps: {
+          config: coreServices.rootConfig,
+          catalog: catalogProcessingExtensionPoint,
+          logger: coreServices.logger,
+          scheduler: coreServices.scheduler,
+          events: eventsServiceRef,
+        },
+        async init({ config, catalog, logger, scheduler, events }) {
+          const gitlabOrgDiscoveryEntityProvider =
+            GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
+              /* highlight-add-next-line */
+              groupNameTransformer: customGroupNameTransformer,
+              logger: loggerToWinstonLogger(logger),
+              events,
+              scheduler,
+            });
+          catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider);
+        },
+      });
+    },
+  });
+```
+
 ## Troubleshooting
 
 **NOTE**: If any groups that are being ingested are empty groups (i.e. do not