chore: add documentation and fix package.json

Signed-off-by: ElaineDeMattosSilvaB <elaine.de-mattos-silva-bezerra@deutschebahn.com>
2024-03-20 22:26:18 +01:00
parent aef7d0e269
commit c3c2591abf
2 changed files with 276 additions and 38 deletions
@@ -11,6 +11,116 @@ entities from GitLab. The entity provider will crawl the GitLab instance and reg
 entities matching the configured paths. This can be useful as an alternative to
 static locations or manually adding things to the catalog.

+This provider can also be configured to ingest GitLab data based on [GitLab Webhooks](https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#configure-a-webhook-in-gitlab). The events currently accepted are:
+
+- [`push`](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#push-events).
+
+## Installation
+
+As this provider is not one of the default providers, you will first need to install
+the gitlab catalog plugin:
+
+```bash
+# From your Backstage root directory
+yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab
+```
+
+### Installation with New Backend System
+
+Then add the following to your backend initialization:
+
+```ts title="packages/backend/src/index.ts
+// optional if you want HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend/alpha'));
+// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha'));
+// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md
+// backend.add(eventsModuleGitlabEventRouter());
+// optional - token validator for the gitlab topic
+// backend.add(eventsModuleGitlabWebhook());
+backend.add(import('@backstage/plugin-catalog-backend-module-gitlab/alpha'));
+```
+
+You need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Further documentation:
+
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+### Installation with Legacy Backend System
+
+#### Installation without Events Support
+
+Add the segment below to `packages/backend/src/plugins/catalog.ts`:
+
+```ts title="packages/backend/src/plugins/catalog.ts"
+/* highlight-add-next-line */
+import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  /* highlight-add-start */
+  builder.addEntityProvider(
+    GitlabDiscoveryEntityProvider.fromConfig(env.config, {
+      logger: env.logger,
+      scheduler: env.scheduler,
+    }),
+  );
+  /* highlight-add-end */
+
+  // ..
+}
+```
+
+#### Installation with Events Support
+
+Please follow the installation instructions at
+
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+Additionally, you need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Set up your provider
+
+```ts title="packages/backend/src/plugins/catalog.ts"
+import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+/* highlight-add-next-line */
+import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
+import { Router } from 'express';
+import { PluginEnvironment } from '../types';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new ScaffolderEntitiesProcessor());
+  /* highlight-add-start */
+  const gitlabProvider = GitlabDiscoveryEntityProvider.fromConfig(env.config, {
+    logger: env.logger,
+    scheduler: env.scheduler,
+  });
+  env.eventBroker.subscribe(gitlabProvider);
+  builder.addEntityProvider(gitlabProvider);
+  /* highlight-add-end */
+  const { processingEngine, router } = await builder.build();
+  await processingEngine.start();
+  return router;
+}
+```
+
+## Configuration
+
 To use the discovery provider, you'll need a GitLab integration
 [set up](locations.md) with a `token`. Then you can add a provider config per group
 to the catalog configuration:
@@ -34,44 +144,8 @@ catalog:
          timeout: { minutes: 3 }
 ```

-As this provider is not one of the default providers, you will first need to install
-the gitlab catalog plugin:
-
-```bash
-# From your Backstage root directory
-yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab
-```
-
-Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`:
-
-```ts title="packages/backend/src/plugins/catalog.ts"
-import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
-
-const builder = await CatalogBuilder.create(env);
-/** ... other processors and/or providers ... */
-builder.addEntityProvider(
-  ...GitlabDiscoveryEntityProvider.fromConfig(env.config, {
-    logger: env.logger,
-    // optional: alternatively, use scheduler with schedule defined in app-config.yaml
-    schedule: env.scheduler.createScheduledTaskRunner({
-      frequency: { minutes: 30 },
-      timeout: { minutes: 3 },
-    }),
-    // optional: alternatively, use schedule
-    scheduler: env.scheduler,
-  }),
-);
-```
-
 ## Alternative processor

-```yaml
-catalog:
-  locations:
-    - type: gitlab-discovery
-      target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml
-```
-
 As alternative to the entity provider `GitlabDiscoveryEntityProvider`
 you can still use the `GitLabDiscoveryProcessor`.

@@ -109,6 +183,15 @@ export default async function createPlugin(
 }
 ```

+And add the following to your app-config.yaml
+
+```yaml
+catalog:
+  locations:
+    - type: gitlab-discovery
+      target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml
+```
+
 If you don't want create location object if file with component definition do not exists in project, you can set the `skipReposWithoutExactFileMatch` option. That can reduce count of request to gitlab with 404 status code.

 If you don't want to create location object if the project is a fork, you can set the `skipForkedRepos` option.
@@ -11,14 +11,57 @@ groups -- directly from GitLab. The result is a hierarchy of
 [`Group`](../../features/software-catalog/descriptor-format.md#kind-group)
 entities that mirrors your org setup.

+This provider can also be configured to ingest GitLab data based on [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html). The events currently accepted are:
+
+- `group_create`
+- `group_destroy`
+- `group_rename`
+- `user_create`
+- `user_destroy`
+- `user_add_to_group`
+- `user_remove_from_group`
+
+## Installation
+
 As this provider is not one of the default providers, you will first need to install the Gitlab provider plugin:

 ```bash
 # From your Backstage root directory
-yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab
+yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab @backstage/plugin-catalog-backend-module-gitlab-org
 ```

-Then add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`:
+### Installation with New Backend System
+
+Then add the following to your backend initialization:
+
+```ts title="packages/backend/src/index.ts
+// optional if you want HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend/alpha'));
+// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events
+// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha'));
+// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md
+// backend.add(eventsModuleGitlabEventRouter());
+// optional - token validator for the gitlab topic
+// backend.add(eventsModuleGitlabWebhook());
+backend.add(import('@backstage/plugin-catalog-backend-module-gitlab-org'));
+```
+
+You need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Further documentation:
+
+- [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html)
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+### Installation with Legacy Backend System
+
+#### Installation without Events Support
+
+Add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`:

 ```ts
 /* packages/backend/src/plugins/catalog.ts */
@@ -41,6 +84,50 @@ builder.addEntityProvider(
 /* highlight-add-end */
 ```

+#### Installation with Events Support
+
+Please follow the installation instructions at
+
+- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md)
+
+Additionally, you need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Set up your provider
+
+```ts title="packages/backend/src/plugins/catalog.ts"
+import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+/* highlight-add-next-line */
+import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
+import { Router } from 'express';
+import { PluginEnvironment } from '../types';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new ScaffolderEntitiesProcessor());
+  /* highlight-add-start */
+  const gitlabOrgProvider = GitlabOrgDiscoveryEntityProvider.fromConfig(
+    env.config,
+    {
+      logger: env.logger,
+      scheduler: env.scheduler,
+    },
+  );
+  env.eventBroker.subscribe(gitlabOrgProvider);
+  builder.addEntityProvider(gitlabOrgProvider);
+  /* highlight-add-end */
+  const { processingEngine, router } = await builder.build();
+  await processingEngine.start();
+  return router;
+}
+```
+
 ## Configuration

 To use the entity provider, you'll need a [Gitlab integration set up](https://backstage.io/docs/integrations/gitlab/locations).
@@ -107,6 +194,74 @@ catalog:
          - allow: [Group, User]
 ```

+### Custom Transformers
+
+You can inject your own transformation logic to help map GitLab API responses
+into Backstage entities. You can do this on the user and group requests to
+enable you to do further processing or updates to these entities.
+
+To enable this you pass a function into the `GitlabOrgDiscoveryEntityProvider`. You can
+pass a `UserTransformer`, a `GroupEntitiesTransformer` or a `GroupNameTransformer` (or all of them)). The function is invoked
+for each item (user or group) that is returned from the API.
+
+The example below uses the groupNameTransformer option to change the metadata.name property of the Backstage Group Entity. Instead of populating it with the usual `group.full_path` data that comes from GitLab, it uses the `group.id`:
+
+```ts
+import { loggerToWinstonLogger } from '@backstage/backend-common';
+import {
+  coreServices,
+  createBackendModule,
+} from '@backstage/backend-plugin-api';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
+import {
+  GitlabOrgDiscoveryEntityProvider,
+  /* highlight-add-next-line */
+  GroupNameTransformerOptions,
+} from '@backstage/plugin-catalog-backend-module-gitlab';
+
+/* highlight-add-start */
+function customGroupNameTransformer(
+  options: GroupNameTransformerOptions,
+): string {
+  return `${options.group.id}`;
+}
+/* highlight-add-end */
+
+/**
+ * Registers the GitlabDiscoveryEntityProvider with the catalog processing extension point.
+ *
+ * @alpha
+ */
+export const catalogModuleGitlabOrgDiscoveryEntityProvider =
+  createBackendModule({
+    pluginId: 'catalog',
+    moduleId: 'gitlabOrgDiscoveryEntityProvider',
+    register(env) {
+      env.registerInit({
+        deps: {
+          config: coreServices.rootConfig,
+          catalog: catalogProcessingExtensionPoint,
+          logger: coreServices.logger,
+          scheduler: coreServices.scheduler,
+          events: eventsServiceRef,
+        },
+        async init({ config, catalog, logger, scheduler, events }) {
+          const gitlabOrgDiscoveryEntityProvider =
+            GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
+              /* highlight-add-next-line */
+              groupNameTransformer: customGroupNameTransformer,
+              logger: loggerToWinstonLogger(logger),
+              events,
+              scheduler,
+            });
+          catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider);
+        },
+      });
+    },
+  });
+```
+
 ## Troubleshooting

 **NOTE**: If any groups that are being ingested are empty groups (i.e. do not