diff --git a/.changeset/eleven-pandas-divide.md b/.changeset/eleven-pandas-divide.md new file mode 100644 index 0000000000..912d34570a --- /dev/null +++ b/.changeset/eleven-pandas-divide.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Added events support for `GitlabDiscoveryEntityProvider` and `GitlabOrgDiscoveryEntityProvider`. diff --git a/.changeset/shy-students-clap.md b/.changeset/shy-students-clap.md new file mode 100644 index 0000000000..0e6244f21d --- /dev/null +++ b/.changeset/shy-students-clap.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab-org': patch +--- + +Added a new `catalog-backend-module-gitlab-org` module which adds the `GitlabOrgDiscoveryEntityProvider` to the catalog's providers using the new backend system. diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index d41f4f8de5..91efe05fee 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -11,28 +11,11 @@ entities from GitLab. The entity provider will crawl the GitLab instance and reg entities matching the configured paths. This can be useful as an alternative to static locations or manually adding things to the catalog. -To use the discovery provider, you'll need a GitLab integration -[set up](locations.md) with a `token`. Then you can add a provider config per group -to the catalog configuration: +This provider can also be configured to ingest GitLab data based on [GitLab Webhooks](https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#configure-a-webhook-in-gitlab). The events currently accepted are: -```yaml -catalog: - providers: - gitlab: - yourProviderId: - host: gitlab-host # Identifies one of the hosts set up in the integrations - branch: main # Optional. Used to discover on a specific branch - fallbackBranch: main # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default - skipForkedRepos: false # Optional. If the project is a fork, skip repository - group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned - entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` - projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything - schedule: # optional; same options as in TaskScheduleDefinition - # supports cron, ISO duration, "human duration" as used in code - frequency: { minutes: 30 } - # supports ISO duration, "human duration" as used in code - timeout: { minutes: 3 } -``` +- [`push`](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#push-events). + +## Installation As this provider is not one of the default providers, you will first need to install the gitlab catalog plugin: @@ -42,15 +25,93 @@ the gitlab catalog plugin: yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab ``` -Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`: +### Installation with New Backend System + +Then add the following to your backend initialization: + +```ts title="packages/backend/src/index.ts +// optional if you want HTTP endpoints to receive external events +// backend.add(import('@backstage/plugin-events-backend/alpha')); +// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events +// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha')); +// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md +// backend.add(eventsModuleGitlabEventRouter()); +// optional - token validator for the gitlab topic +// backend.add(eventsModuleGitlabWebhook()); +backend.add(import('@backstage/plugin-catalog-backend-module-gitlab/alpha')); +``` + +You need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Further documentation: + +- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md) + +### Installation with Legacy Backend System + +#### Installation without Events Support + +Add the segment below to `packages/backend/src/plugins/catalog.ts`: ```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; -const builder = await CatalogBuilder.create(env); -/** ... other processors and/or providers ... */ -builder.addEntityProvider( - ...GitlabDiscoveryEntityProvider.fromConfig(env.config, { +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + /* highlight-add-start */ + builder.addEntityProvider( + ...GitlabDiscoveryEntityProvider.fromConfig(env.config, { + logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // optional: alternatively, use schedule + scheduler: env.scheduler, + }), + ); + /* highlight-add-end */ + // .. +} +``` + +#### Installation with Events Support + +Please follow the installation instructions at + +- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md) + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```ts title="packages/backend/src/plugins/catalog.ts" +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; +/* highlight-add-next-line */ +import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; +import { Router } from 'express'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + builder.addProcessor(new ScaffolderEntitiesProcessor()); + /* highlight-add-start */ + const gitlabProvider = GitlabDiscoveryEntityProvider.fromConfig(env.config, { logger: env.logger, // optional: alternatively, use scheduler with schedule defined in app-config.yaml schedule: env.scheduler.createScheduledTaskRunner({ @@ -59,19 +120,45 @@ builder.addEntityProvider( }), // optional: alternatively, use schedule scheduler: env.scheduler, - }), -); + }); + env.eventBroker.subscribe(gitlabProvider); + builder.addEntityProvider(gitlabProvider); + /* highlight-add-end */ + const { processingEngine, router } = await builder.build(); + await processingEngine.start(); + return router; +} +``` + +## Configuration + +To use the discovery provider, you'll need a GitLab integration +[set up](locations.md) with a `token`. Then you can add a provider config per group +to the catalog configuration. + +> > NOTE: if you are using the New Backend System, the `schedule` has to be setup in the config, as shown below. + +```yaml title="app-config.yaml" +catalog: + providers: + gitlab: + yourProviderId: + host: gitlab-host # Identifies one of the hosts set up in the integrations + branch: main # Optional. Used to discover on a specific branch + fallbackBranch: master # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default + skipForkedRepos: false # Optional. If the project is a fork, skip repository + group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned + entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` + projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything + schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` ## Alternative processor -```yaml -catalog: - locations: - - type: gitlab-discovery - target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml -``` - As alternative to the entity provider `GitlabDiscoveryEntityProvider` you can still use the `GitLabDiscoveryProcessor`. @@ -109,6 +196,15 @@ export default async function createPlugin( } ``` +And add the following to your app-config.yaml + +```yaml +catalog: + locations: + - type: gitlab-discovery + target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml +``` + If you don't want create location object if file with component definition do not exists in project, you can set the `skipReposWithoutExactFileMatch` option. That can reduce count of request to gitlab with 404 status code. If you don't want to create location object if the project is a fork, you can set the `skipForkedRepos` option. diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index f2a483c9d1..14587461e5 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -11,34 +11,134 @@ groups -- directly from GitLab. The result is a hierarchy of [`Group`](../../features/software-catalog/descriptor-format.md#kind-group) entities that mirrors your org setup. +This provider can also be configured to ingest GitLab data based on [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html). The events currently accepted are: + +- `group_create` +- `group_destroy` +- `group_rename` +- `user_create` +- `user_destroy` +- `user_add_to_group` +- `user_remove_from_group` + +## Installation + As this provider is not one of the default providers, you will first need to install the Gitlab provider plugin: ```bash # From your Backstage root directory -yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab @backstage/plugin-catalog-backend-module-gitlab-org ``` -Then add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`: +### Installation with New Backend System + +Then add the following to your backend initialization: + +```ts title="packages/backend/src/index.ts +// optional if you want HTTP endpoints to receive external events +// backend.add(import('@backstage/plugin-events-backend/alpha')); +// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events +// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha')); +// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md +// backend.add(eventsModuleGitlabEventRouter()); +// optional - token validator for the gitlab topic +// backend.add(eventsModuleGitlabWebhook()); +backend.add(import('@backstage/plugin-catalog-backend-module-gitlab-org')); +``` + +You need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Further documentation: + +- [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html) +- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md) + +### Installation with Legacy Backend System + +#### Installation without Events Support + +Add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`: ```ts /* packages/backend/src/plugins/catalog.ts */ /* highlight-add-next-line */ import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; -const builder = await CatalogBuilder.create(env); -/** ... other processors and/or providers ... */ -/* highlight-add-start */ -builder.addEntityProvider( - ...GitlabOrgDiscoveryEntityProvider.fromConfig(env.config, { - logger: env.logger, - // optional: alternatively, use scheduler with schedule defined in app-config.yaml - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 30 }, - timeout: { minutes: 3 }, +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + /** ... other processors and/or providers ... */ + /* highlight-add-start */ + builder.addEntityProvider( + ...GitlabOrgDiscoveryEntityProvider.fromConfig(env.config, { + logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // optional: alternatively, use schedule + scheduler: env.scheduler, }), - }), -); -/* highlight-add-end */ + ); + /* highlight-add-end */ + // .. +} +``` + +#### Installation with Events Support + +Please follow the installation instructions at + +- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md) + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```ts title="packages/backend/src/plugins/catalog.ts" +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; +/* highlight-add-next-line */ +import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; +import { Router } from 'express'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + builder.addProcessor(new ScaffolderEntitiesProcessor()); + /* highlight-add-start */ + const gitlabOrgProvider = GitlabOrgDiscoveryEntityProvider.fromConfig( + env.config, + { + logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // optional: alternatively, use schedule + scheduler: env.scheduler, + }, + ); + env.eventBroker.subscribe(gitlabOrgProvider); + builder.addEntityProvider(gitlabOrgProvider); + /* highlight-add-end */ + const { processingEngine, router } = await builder.build(); + await processingEngine.start(); + return router; +} ``` ## Configuration @@ -58,6 +158,8 @@ amount of data, this can take significant time and resources. The token used must have the `read_api` scope, and the Users and Groups fetched will be those visible to the account which provisioned the token. +> > NOTE: if you are using the New Backend System, the `schedule` has to be setup in the config, as shown below. + ```yaml catalog: providers: @@ -66,7 +168,13 @@ catalog: host: gitlab.com orgEnabled: true group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) + allowInherited: true # Allow groups to be ingested even if there are no direct members. groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything + schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System. + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` ### Groups @@ -107,6 +215,74 @@ catalog: - allow: [Group, User] ``` +### Custom Transformers + +You can inject your own transformation logic to help map GitLab API responses +into Backstage entities. You can do this on the user and group requests to +enable you to do further processing or updates to these entities. + +To enable this you pass a function into the `GitlabOrgDiscoveryEntityProvider`. You can +pass a `UserTransformer`, a `GroupEntitiesTransformer` or a `GroupNameTransformer` (or all of them)). The function is invoked +for each item (user or group) that is returned from the API. + +The example below uses the groupNameTransformer option to change the metadata.name property of the Backstage Group Entity. Instead of populating it with the usual `group.full_path` data that comes from GitLab, it uses the `group.id`: + +```ts +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; +import { + GitlabOrgDiscoveryEntityProvider, + /* highlight-add-next-line */ + GroupNameTransformerOptions, +} from '@backstage/plugin-catalog-backend-module-gitlab'; + +/* highlight-add-start */ +function customGroupNameTransformer( + options: GroupNameTransformerOptions, +): string { + return `${options.group.id}`; +} +/* highlight-add-end */ + +/** + * Registers the GitlabDiscoveryEntityProvider with the catalog processing extension point. + * + * @alpha + */ +export const catalogModuleGitlabOrgDiscoveryEntityProvider = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'gitlabOrgDiscoveryEntityProvider', + register(env) { + env.registerInit({ + deps: { + config: coreServices.rootConfig, + catalog: catalogProcessingExtensionPoint, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + events: eventsServiceRef, + }, + async init({ config, catalog, logger, scheduler, events }) { + const gitlabOrgDiscoveryEntityProvider = + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + /* highlight-add-next-line */ + groupNameTransformer: customGroupNameTransformer, + logger: loggerToWinstonLogger(logger), + events, + scheduler, + }); + catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider); + }, + }); + }, + }); +``` + ## Troubleshooting **NOTE**: If any groups that are being ingested are empty groups (i.e. do not diff --git a/plugins/catalog-backend-module-gitlab-org/.eslintrc.js b/plugins/catalog-backend-module-gitlab-org/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-gitlab-org/README.md b/plugins/catalog-backend-module-gitlab-org/README.md new file mode 100644 index 0000000000..0b86b8d815 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/README.md @@ -0,0 +1,8 @@ +# Catalog Backend Module for GitLab Organizational Data + +This is an extension module to the plugin-catalog-backend plugin that provides group and user entities by scrapping or receiving events from a GitLab instance. + +## Getting started + +See [Backstage documentation](https://backstage.io/docs/integrations/gitlab/org) for details on how to install +and configure the plugin. diff --git a/plugins/catalog-backend-module-gitlab-org/api-report.md b/plugins/catalog-backend-module-gitlab-org/api-report.md new file mode 100644 index 0000000000..2a36f49206 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/api-report.md @@ -0,0 +1,11 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-gitlab-org" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; + +// @public +const catalogModuleGitlabOrgDiscoveryEntityProvider: () => BackendFeature; +export default catalogModuleGitlabOrgDiscoveryEntityProvider; +``` diff --git a/plugins/catalog-backend-module-gitlab-org/catalog-info.yaml b/plugins/catalog-backend-module-gitlab-org/catalog-info.yaml new file mode 100644 index 0000000000..1846e3a9df --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-catalog-backend-module-gitlab-org + title: '@backstage/plugin-catalog-backend-module-gitlab-org' + description: The gitlab-org backend module for the catalog plugin. +spec: + lifecycle: experimental + type: backstage-backend-plugin-module + owner: maintainers diff --git a/plugins/catalog-backend-module-gitlab-org/package.json b/plugins/catalog-backend-module-gitlab-org/package.json new file mode 100644 index 0000000000..7d92fda325 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/package.json @@ -0,0 +1,47 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-gitlab-org", + "version": "0.0.0", + "description": "The gitlab-org backend module for the catalog plugin.", + "backstage": { + "role": "backend-plugin-module" + }, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-gitlab-org" + }, + "license": "Apache-2.0", + "main": "src/index.ts", + "types": "src/index.ts", + "files": [ + "dist" + ], + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^" + }, + "devDependencies": { + "@backstage/backend-tasks": "workspace:^", + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@backstage/plugin-events-backend-test-utils": "workspace:^", + "luxon": "^3.0.0" + } +} diff --git a/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts new file mode 100644 index 0000000000..568c91c8e2 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createServiceFactory } from '@backstage/backend-plugin-api'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { TestEventsService } from '@backstage/plugin-events-backend-test-utils'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; +import { Duration } from 'luxon'; +import { catalogModuleGitlabOrgDiscoveryEntityProvider } from './catalogModuleGitlabOrgDiscoveryEntityProvider'; + +describe('catalogModuleGitlabOrgDiscoveryEntityProvider', () => { + it('should register provider at the catalog extension point', async () => { + const events = new TestEventsService(); + const eventsServiceFactory = createServiceFactory({ + service: eventsServiceRef, + deps: {}, + async factory({}) { + return events; + }, + }); + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const connection = jest.fn() as unknown as EntityProviderConnection; + const runner = jest.fn(); + const scheduler = mockServices.scheduler.mock({ + createScheduledTaskRunner(schedule) { + usedSchedule = schedule; + return { run: runner }; + }, + }); + + const config = { + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + orgEnabled: true, + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }; + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + features: [ + eventsServiceFactory(), + catalogModuleGitlabOrgDiscoveryEntityProvider(), + mockServices.rootConfig.factory({ data: config }), + mockServices.logger.factory(), + scheduler.factory, + ], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(runner).not.toHaveBeenCalled(); + + const provider = addedProviders!.pop()!; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + await provider.connect(connection); + expect(events.subscribed).toHaveLength(1); + expect(events.subscribed[0].id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + expect(runner).toHaveBeenCalledTimes(1); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts new file mode 100644 index 0000000000..ffb2cfb629 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; + +/** + * Registers the GitlabOrgDiscoveryEntityProvider with the catalog processing extension point. + * + * @public + */ +export const catalogModuleGitlabOrgDiscoveryEntityProvider = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'gitlabOrgDiscoveryEntityProvider', + register(env) { + env.registerInit({ + deps: { + config: coreServices.rootConfig, + catalog: catalogProcessingExtensionPoint, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + events: eventsServiceRef, + }, + async init({ config, catalog, logger, scheduler, events }) { + const gitlabOrgDiscoveryEntityProvider = + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger: loggerToWinstonLogger(logger), + events, + scheduler, + }); + catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider); + }, + }); + }, + }); diff --git a/plugins/catalog-backend-module-gitlab-org/src/index.ts b/plugins/catalog-backend-module-gitlab-org/src/index.ts new file mode 100644 index 0000000000..bbd5f1f4c3 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/src/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The gitlab-org backend module for the catalog plugin. + * + * @packageDocumentation + */ +export { catalogModuleGitlabOrgDiscoveryEntityProvider as default } from './catalogModuleGitlabOrgDiscoveryEntityProvider'; diff --git a/plugins/catalog-backend-module-gitlab/CHANGELOG.md b/plugins/catalog-backend-module-gitlab/CHANGELOG.md index 4e8493d5d6..92f0ea7f21 100644 --- a/plugins/catalog-backend-module-gitlab/CHANGELOG.md +++ b/plugins/catalog-backend-module-gitlab/CHANGELOG.md @@ -385,8 +385,8 @@ ref: - https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api - https://docs.gitlab.com/ee/api/members.html#limitations + + - 890e3b5ad4: Make sure to include the error message when ingestion fails - 0b55f773a7: Removed some unused dependencies @@ -417,8 +417,8 @@ ref: - https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api - https://docs.gitlab.com/ee/api/members.html#limitations + + - 0b55f773a7: Removed some unused dependencies - Updated dependencies @@ -1211,7 +1211,7 @@ - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. Please find how to configure the schedule at the config at - https://backstage.io/docs/integrations/gitlab/discovery + - 4c9f7847e4: Updated dependency `msw` to `^0.48.0` while moving it to be a dev dependency. - Updated dependencies @@ -1250,7 +1250,7 @@ - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. Please find how to configure the schedule at the config at - https://backstage.io/docs/integrations/gitlab/discovery + - Updated dependencies - @backstage/backend-common@0.16.0-next.0 diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index cd9ade4bfb..d7fcd1b94d 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -8,6 +8,7 @@ import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { EventsService } from '@backstage/plugin-events-node'; import { GitLabIntegrationConfig } from '@backstage/integration'; import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/plugin-catalog-node'; @@ -26,13 +27,13 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; }, ): GitlabDiscoveryEntityProvider[]; // (undocumented) getProviderName(): string; - // (undocumented) refresh(logger: LoggerService): Promise; } @@ -81,6 +82,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; userTransformer?: UserTransformer; @@ -103,6 +105,7 @@ export type GitlabProviderConfig = { projectPattern: RegExp; userPattern: RegExp; groupPattern: RegExp; + allowInherited?: boolean; orgEnabled?: boolean; schedule?: TaskScheduleDefinition; skipForkedRepos?: boolean; diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 75bccd4faa..99fdde8346 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -1,18 +1,67 @@ { - "name": "@backstage/plugin-catalog-backend-module-gitlab", + "backstage": { + "role": "backend-plugin-module" + }, + "configSchema": "config.d.ts", + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/integration": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", + "@gitbeaker/rest": "^40.0.3", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7", + "uuid": "^9.0.0" + }, "description": "A Backstage catalog backend module that helps integrate towards GitLab", - "version": "0.3.15", - "main": "src/index.ts", - "types": "src/index.ts", - "license": "Apache-2.0", - "publishConfig": { - "access": "public" + "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@backstage/plugin-events-backend-test-utils": "workspace:^", + "@types/lodash": "^4.14.151", + "@types/uuid": "^9.0.0", + "luxon": "^3.0.0", + "msw": "^1.0.0" }, "exports": { ".": "./src/index.ts", "./alpha": "./src/alpha.ts", "./package.json": "./package.json" }, + "files": [ + "config.d.ts", + "dist" + ], + "homepage": "https://backstage.io", + "keywords": [ + "backstage" + ], + "license": "Apache-2.0", + "main": "src/index.ts", + "name": "@backstage/plugin-catalog-backend-module-gitlab", + "publishConfig": { + "access": "public" + }, + "repository": { + "directory": "plugins/catalog-backend-module-gitlab", + "type": "git", + "url": "https://github.com/backstage/backstage" + }, + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "postpack": "backstage-cli package postpack", + "prepack": "backstage-cli package prepack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, + "types": "src/index.ts", "typesVersions": { "*": { "alpha": [ @@ -23,50 +72,5 @@ ] } }, - "backstage": { - "role": "backend-plugin-module" - }, - "homepage": "https://backstage.io", - "repository": { - "type": "git", - "url": "https://github.com/backstage/backstage", - "directory": "plugins/catalog-backend-module-gitlab" - }, - "keywords": [ - "backstage" - ], - "scripts": { - "start": "backstage-cli package start", - "build": "backstage-cli package build", - "lint": "backstage-cli package lint", - "test": "backstage-cli package test", - "prepack": "backstage-cli package prepack", - "postpack": "backstage-cli package postpack", - "clean": "backstage-cli package clean" - }, - "dependencies": { - "@backstage/backend-common": "workspace:^", - "@backstage/backend-plugin-api": "workspace:^", - "@backstage/backend-tasks": "workspace:^", - "@backstage/catalog-model": "workspace:^", - "@backstage/config": "workspace:^", - "@backstage/integration": "workspace:^", - "@backstage/plugin-catalog-node": "workspace:^", - "lodash": "^4.17.21", - "node-fetch": "^2.6.7", - "uuid": "^9.0.0" - }, - "devDependencies": { - "@backstage/backend-test-utils": "workspace:^", - "@backstage/cli": "workspace:^", - "@types/lodash": "^4.14.151", - "@types/uuid": "^9.0.0", - "luxon": "^3.0.0", - "msw": "^1.0.0" - }, - "files": [ - "config.d.ts", - "dist" - ], - "configSchema": "config.d.ts" + "version": "0.3.15-next.1" } diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts new file mode 100644 index 0000000000..00d4994e94 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts @@ -0,0 +1,537 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { graphql, rest } from 'msw'; +import { + all_groups_response, + all_projects_response, + all_saas_users_response, + all_users_response, + apiBaseUrl, + apiBaseUrlSaas, + graphQlBaseUrl, + paged_endpoint, + saasGraphQlBaseUrl, + some_endpoint, + unhealthy_endpoint, + userID, +} from './mocks'; + +const httpHandlers = [ + /** + * Project REST endpoint mocks + */ + + // fetch all projects in an instance + rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => { + return res(ctx.set('x-next-page', ''), ctx.json(all_projects_response)); + }), + + rest.get(`${apiBaseUrl}/projects/42`, (_, res, ctx) => { + return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); + }), + + // testing non existing file + rest.get( + `${apiBaseUrl}/projects/test-group%2Ftest-repo1/repository/files/catalog-info.yaml`, + (_, res, ctx) => { + return res(ctx.status(400), ctx.json({ error: 'Not found' })); + }, + ), + + /** + * Group REST endpoint mocks + */ + + rest.get(`${apiBaseUrl}/groups`, (_req, res, ctx) => { + return res(ctx.set('x-next-page', ''), ctx.json(all_groups_response)); + }), + + rest.get(`${apiBaseUrl}/groups/42`, (_, res, ctx) => { + return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); + }), + + rest.get(`${apiBaseUrlSaas}/groups/group1/members/all`, (_req, res, ctx) => { + return res(ctx.json(all_saas_users_response)); + }), + + /** + * Users REST endpoint mocks + */ + + rest.get(`${apiBaseUrl}/users`, (_req, res, ctx) => { + return res(ctx.set('x-next-page', ''), ctx.json(all_users_response)); + }), + + rest.get(`${apiBaseUrl}/users/${userID}`, (_, res, ctx) => { + return res(ctx.json(all_users_response.find(user => user.id === userID))); + }), + + rest.get(`${apiBaseUrl}/users/42`, (_, res, ctx) => { + return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); + }), + + /** + * others + */ + + // mock a 4 page response + rest.get(`${apiBaseUrl}${paged_endpoint}`, (req, res, ctx) => { + const page = req.url.searchParams.get('page'); + const currentPage = page ? Number(page) : 1; + const fakePageCount = 4; + + return res( + // set next page number header if page requested is less than count + ctx.set( + 'x-next-page', + currentPage < fakePageCount ? String(currentPage + 1) : '', + ), + ctx.json([{ someContentOfPage: currentPage }]), + ); + }), + + rest.get(`${apiBaseUrl}${unhealthy_endpoint}`, (_, res, ctx) => { + return res(ctx.status(400), ctx.json({ error: 'some error' })); + }), + + rest.get(`${apiBaseUrl}${some_endpoint}`, (req, res, ctx) => { + return res(ctx.json([{ endpoint: req.url.toString() }])); + }), +]; + +// dynamic handlers + +const httpGroupFindByIdDynamic = all_groups_response.map(group => { + return rest.get(`${apiBaseUrl}/groups/${group.id}`, (_, res, ctx) => { + return res(ctx.json(all_groups_response.find(g => g.id === group.id))); + }); +}); + +const httpGroupListDescendantProjectsById = all_groups_response.map(group => { + return rest.get( + `${apiBaseUrl}/groups/${group.id}/projects`, + (_, res, ctx) => { + const projectsInGroup = all_projects_response.filter(p => + p.path_with_namespace?.includes(group.name), + ); + + return res(ctx.json(projectsInGroup)); + }, + ); +}); + +const httpGroupListDescendantProjectsByName = all_groups_response.map(group => { + return rest.get( + `${apiBaseUrl}/groups/${group.name}/projects`, + (_, res, ctx) => { + const projectsInGroup = all_projects_response.filter(p => + p.path_with_namespace?.includes(group.name), + ); + + return res(ctx.json(projectsInGroup)); + }, + ); +}); +const httpGroupFindByNameDynamic = all_groups_response.map(group => { + return rest.get(`${apiBaseUrl}/groups/${group.name}`, (_, res, ctx) => { + return res(ctx.json(all_groups_response.find(g => g.name === group.name))); + }); +}); +const httpProjectFindByIdDynamic = all_projects_response.map(project => { + return rest.get(`${apiBaseUrl}/projects/${project.id}`, (_, res, ctx) => { + return res(ctx.json(all_projects_response.find(p => p.id === project.id))); + }); +}); +const httpProjectCatalogDynamic = all_projects_response.map(project => { + const path: string = project.path_with_namespace + ? project.path_with_namespace!.replace(/\//g, '%2F') + : `${project.path_with_namespace}%2F${project.name}`; + + return rest.head( + `${apiBaseUrl}/projects/${path}/repository/files/catalog-info.yaml`, + (req, res, ctx) => { + const branch = req.url.searchParams.get('ref'); + if (branch === project.default_branch) { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ); +}); + +/** + * GraphQL endpoint mocks + */ + +const graphqlHandlers = [ + graphql + .link(graphQlBaseUrl) + .query('getGroupMembers', async (req, res, ctx) => { + const { group, relations } = req.variables; + + if (group === 'group1' && relations.includes('DIRECT')) { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + // group with no associated members + if (group === 'group3' && relations.includes('DIRECT')) { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [{}], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + if (group === 'saas-multi-user-group') { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + publicEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + + if (group === 'non-existing-group' || group === '') { + return res( + ctx.data({ + group: { + groupMembers: { + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + + if (group === 'multi-page' && relations.includes('DIRECT')) { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: req.variables.endCursor + ? [{ user: { id: 'gid://gitlab/User/2' } }] + : [{ user: { id: 'gid://gitlab/User/1' } }], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ); + } + + if (group === 'multi-page-saas') { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: req.variables.endCursor + ? [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ] + : [ + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + publicEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ); + } + + if (group === 'error-group') { + return res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ); + } + + return res(ctx.status(200)); + }), + + graphql + .link(graphQlBaseUrl) + .query('listDescendantGroups', async (req, res, ctx) => { + const { group } = req.variables; + + if (group === 'error-group') { + return res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ); + } + + if (group === 'group-with-parent') { + return res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/1', + name: 'group-with-parent', + description: 'description1', + fullPath: 'path/group-with-parent', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + if (group === 'root') { + return res( + ctx.data({ + group: { + descendantGroups: { + nodes: req.variables.endCursor + ? [ + { + id: 'gid://gitlab/Group/1', + name: 'group1', + description: 'description1', + fullPath: 'path/group1', + parent: { + id: '123', + }, + }, + ] + : [ + { + id: 'gid://gitlab/Group/2', + name: 'group2', + description: 'description2', + fullPath: 'path/group2', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ); + } + + if (group === 'non-existing-group') { + return res( + ctx.data({ + group: {}, + }), + ); + } + return res(ctx.status(200)); + }), + + graphql + .link(saasGraphQlBaseUrl) + .query('getGroupMembers', async (req, res, ctx) => { + const { group } = req.variables; + + if (group === 'saas-multi-user-group') { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + publicEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + + if (group === '') { + return res(ctx.data({})); + } + + if (group === 'error-group') { + return res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ); + } + + return res(ctx.status(200)); + }), + + graphql + .link(saasGraphQlBaseUrl) + .query('listDescendantGroups', async (_, res, ctx) => { + return res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/456', + name: 'group1', + description: 'Group1', + fullPath: 'group1/group1', + parent: { + id: 'gid://gitlab/Group/123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + }), +]; + +export const handlers = [ + ...httpHandlers, + ...httpProjectFindByIdDynamic, + ...httpProjectCatalogDynamic, + ...httpGroupFindByIdDynamic, + ...httpGroupFindByNameDynamic, + ...httpGroupListDescendantProjectsById, + ...httpGroupListDescendantProjectsByName, + ...graphqlHandlers, +]; diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts new file mode 100644 index 0000000000..5a078ab689 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts @@ -0,0 +1,1769 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { locationSpecToMetadataName } from '@backstage/plugin-catalog-node'; +import { EventParams } from '@backstage/plugin-events-node'; +import { GitLabGroup, GitLabProject, GitLabUser } from '../lib/types'; + +type MockObject = Record; + +/** + * General + */ +export const apiBaseUrl: string = 'https://example.com/api/v4'; +export const apiBaseUrlSaas: string = 'https://gitlab.com/api/v4'; +export const graphQlBaseUrl = 'https://example.com/api/graphql'; +export const saasGraphQlBaseUrl = 'https://gitlab.com/api/graphql'; +export const groupName: string = 'group1'; +export const groupID: number = 1; +export const userID: number = 1; +export const projectID: number = 1; + +/** + * Endpoints + */ + +export const paged_endpoint: string = `/paged-endpoint`; +export const some_endpoint: string = `/some-endpoint`; +export const unhealthy_endpoint = `/unhealthy-endpoint`; + +/** + * Integration Configurations + */ + +export const config_self_managed: MockObject = { + host: 'example.com', + token: 'test-token', + apiBaseUrl: 'https://example.com/api/v4', + baseUrl: 'https://example.com', +}; + +export const config_saas: MockObject = { + host: 'gitlab.com', + token: 'test-token', + apiBaseUrl: 'https://gitlab.com/api/v4', + baseUrl: 'https://gitlab.com', +}; + +export const config_no_org_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_disabled_org_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + skipForkedRepos: false, + orgEnabled: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_saas_no_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + skipForkedRepos: false, + orgEnabled: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_github_host: MockObject = { + integrations: { + github: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_single_integration_branch: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + branch: 'main', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_single_integration_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'nonMatchingGroup', + branch: 'main', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_fallbackBranch_branch: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + fallbackBranch: 'staging', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_single_integration_skip_forks: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + skipForkedRepos: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_no_schedule: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + orgEnabled: true, + skipForkedRepos: true, + }, + }, + }, + }, +}; + +export const config_single_integration_project_pattern: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + projectPattern: 'test-repo', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_no_schedule_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + projectPattern: 'test-repo', + skipForkedRepos: true, + }, + }, + }, + }, +}; + +export const config_unmatched_project_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'my-other-group', + projectPattern: 'my-other-project', + skipForkedRepos: true, + }, + }, + }, + }, +}; + +export const config_double_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + }, + 'second-test': { + host: 'example.com', + group: 'second-group', + }, + }, + }, + }, +}; + +export const config_org_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_userPattern_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + userPattern: '^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+.[a-zA-Z]{2,}$', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_saas: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + group: 'group1', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_no_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_saas_no_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_saas_sched: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + group: 'group1', + groupPattern: '^group.*', + orgEnabled: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_org_double_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + orgEnabled: true, + }, + 'second-test': { + host: 'example.com', + group: 'second-group', + orgEnabled: true, + }, + }, + }, + }, +}; + +/** + * GitLab API responses + */ + +export const all_projects_response: GitLabProject[] = [ + { + id: 1, + description: 'Project One Description', + name: 'test-repo1', + default_branch: 'main', + path: 'test-repo1', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo1', + path_with_namespace: 'group1/test-repo1', + }, + { + id: 2, + description: 'Project Two Description', + name: 'test-repo2', + default_branch: 'prd', + path: 'test-repo2', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo2', + path_with_namespace: 'group1/test-repo2', + }, + // unmatched project + { + id: 3, + description: 'Project Three Description', + name: 'repo3', + default_branch: 'main', + path: 'repo3', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/repo3', + path_with_namespace: 'group1/repo3', + }, + // forked project + { + id: 4, + description: 'Project Four Description', + name: 'test-repo4-forked', + default_branch: 'main', + path: 'test-repo4-forked', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo4-forked', + path_with_namespace: 'group1/test-repo4-forked', + forked_from_project: { id: 13083 }, + }, + // fallBack branch + { + id: 5, + description: 'Project Five Description', + name: 'test-repo5-staging', + default_branch: 'staging', + path: 'test-repo5-staging', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo5-staging', + path_with_namespace: 'group1/test-repo5-staging', + }, + // diffrent group + { + id: 6, + description: 'Project Six Description', + name: 'test-repo6', + default_branch: 'main', + path: 'test-repo6', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo6', + path_with_namespace: 'awesome-group/test-repo6', + }, +]; + +export const all_users_response: GitLabUser[] = [ + { + id: 1, + username: 'JohnDoe', + name: 'John Doe', + state: 'active', + email: 'john.doe@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/john_doe', + }, + { + id: 2, + username: 'JaneDoe', + name: 'Jane Doe', + state: 'active', + email: 'jane.doe@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/jane_doe', + }, + { + id: 3, + username: 'MarySmith', + name: 'Mary Smith', + state: 'active', + email: 'mary.smith@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/mary_smith', + }, + // inactive user + { + id: 4, + username: 'LuigiMario', + name: 'Luigi Mario', + state: 'inactive', + email: 'luigi.mario@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/luigi_mario', + }, + // malfomed email address + { + id: 5, + username: 'MarioMario', + name: 'Mario Mario', + state: 'active', + email: 'mario.mario-company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/mario_mario', + }, +]; + +export const all_saas_users_response: MockObject[] = [ + { + access_level: 30, + created_at: '2023-07-17T08:58:34.984Z', + expires_at: null, + id: 12, + username: 'testuser1', + name: 'Test User 1', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.com/testuser1', + email: 'testuser1@example.com', + group_saml_identity: { + provider: 'group_saml', + extern_uid: '51', + saml_provider_id: 1, + }, + is_using_seat: true, + membership_state: 'active', + }, + { + access_level: 30, + created_at: '2023-07-19T08:58:34.984Z', + expires_at: null, + id: 34, + username: 'testuser2', + name: 'Test User 2', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.com/testuser2', + email: 'testuser2@example.com', + group_saml_identity: { + provider: 'group_saml', + extern_uid: '52', + saml_provider_id: 1, + }, + is_using_seat: true, + membership_state: 'active', + }, + { + access_level: 50, + created_at: '2023-07-15T08:58:34.984Z', + expires_at: '2023-10-26', + id: 54, + username: 'group_100_bot_23dc8057bef66e05181f39be4652577c', + name: 'Token Bot', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: + 'https://gitlab.com/group_100_bot_23dc8057bef66e05181f39be4652577c', + group_saml_identity: null, + is_using_seat: false, + membership_state: 'active', + }, +]; + +export const all_groups_response: GitLabGroup[] = [ + { + id: 1, + name: 'group1', + description: '', + full_path: 'group1', + }, + { + id: 2, + name: 'group2', + description: '', + full_path: 'group2', + }, + { + id: 3, + name: 'group3', + description: '', + full_path: 'group3', + }, + { + id: 4, + name: 'group1', + description: '', + full_path: 'group-new/group1', + }, + { + id: 5, + name: 'nonMatchingGroup', + description: '', + full_path: 'parent1/nonMatchingGroup', + }, +]; + +export const group_with_parent: MockObject[] = [ + { + id: 1, + name: 'group-with-parent', + description: 'description1', + full_path: 'path/group-with-parent', + parent_id: 123, + }, +]; + +export const expectedSaasMember: MockObject[] = [ + { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }, + { + id: 2, + username: 'user2', + email: 'user2@example.com', + name: 'user2', + state: 'active', + web_url: 'user2.com', + avatar_url: 'user2', + }, +]; + +export const expectedSaasGroup: MockObject[] = [ + { + id: 1, + name: 'group1', + description: 'description1', + full_path: 'path/group1', + parent_id: 123, + }, + { + id: 2, + name: 'group2', + description: 'description2', + full_path: 'path/group2', + parent_id: 123, + }, +]; + +/** + * GitLab Events + */ + +const added_commits: MockObject[] = [ + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: ['catalog-info.yaml'], + modified: [], + removed: [], + }, + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: ['cool-folder-1/cool-folder-2/catalog-info.yaml'], + modified: [], + removed: [], + }, +]; + +const removed_commits: MockObject[] = [ + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: [], + modified: [], + removed: ['catalog-info.yaml'], + }, +]; + +const modified_commits: MockObject[] = [ + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: [], + modified: ['catalog-info.yaml'], + removed: [], + }, +]; + +export const group_destroy_event: EventParams = { + topic: 'gitlab.group_destroy', + eventPayload: { + event_name: 'group_destroy', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'group3', + path: 'group3', + full_path: 'group3', + group_id: 123, + }, +}; + +export const group_destroy_event_unmatched: EventParams = { + topic: 'gitlab.group_destroy', + eventPayload: { + event_name: 'group_destroy', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'my-awesome-group', + path: 'my-awesome-group', + full_path: 'parent/my-awesome-group', + group_id: 123, + }, +}; + +export const group_rename_event: EventParams = { + topic: 'gitlab.group_rename', + eventPayload: { + event_name: 'group_rename', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'group1', // this is the displayname + path: 'group-new', + full_path: 'group-new/group1', + old_path: 'group-old', + old_full_path: 'group-old/group1', + group_id: 4, + }, +}; + +export const group_create_event: EventParams = { + topic: 'gitlab.group_create', + eventPayload: { + event_name: 'group_create', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'group3', + path: 'group3', + full_path: 'group3', + group_id: 3, + }, +}; +export const group_create_event_unmatched: EventParams = { + topic: 'gitlab.group_create', + eventPayload: { + event_name: 'group_create', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'nonMatchingGroup', + path: 'nonMatchingGroup', + full_path: 'parent1/nonMatchingGroup', + group_id: 5, + }, +}; + +export const user_create_event: EventParams = { + topic: 'gitlab.user_create', + eventPayload: { + event_name: 'user_create', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + email: 'john.doe@company.com', + name: 'John Doe', + username: 'johndoe', + user_id: 1, + }, +}; +export const user_destroy_event: EventParams = { + topic: 'gitlab.user_destroy', + eventPayload: { + event_name: 'user_destroy', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + email: 'john.doe@company.com', + name: 'John Doe', + username: 'johndoe', + user_id: 1, + }, +}; + +export const user_add_to_group_event: EventParams = { + topic: 'gitlab.user_add_to_group', + eventPayload: { + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + group_name: 'group1', + group_path: 'my-groups/group1', + group_id: 1, + user_username: 'JohnDoe', + user_name: 'John Doe', + user_email: 'john.doe@company.com', + user_id: 1, + group_access: 'Owner', + expires_at: null, + group_plan: null, + event_name: 'user_add_to_group', + }, +}; + +export const user_add_to_group_event_mismatched: EventParams = { + topic: 'gitlab.user_add_to_group', + eventPayload: { + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + group_name: 'nonMatchingGroup', + group_path: 'parent1/nonMatchingGroup', + group_id: 5, + user_username: 'JohnDoe', + user_name: 'John Doe', + user_email: 'john.doe@company.com', + user_id: 1, + group_access: 'Owner', + expires_at: null, + group_plan: null, + event_name: 'user_add_to_group', + }, +}; + +export const user_remove_from_group_event: EventParams = { + topic: 'gitlab.user_remove_from_group', + eventPayload: { + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + group_name: 'group3', + group_path: 'my-groups/group3', + group_id: 3, + user_username: 'user1', + user_name: 'John Doe', + user_email: 'john.doe@company.com', + user_id: 1, + group_access: 'Owner', + expires_at: null, + group_plan: null, + event_name: 'user_remove_from_group', + }, +}; + +export const push_add_event: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 1, + project: { + name: 'test-repo', + path_with_namespace: 'group1/test-repo1', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo1', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo1', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: added_commits, + total_commits_count: 2, + repository: { + name: 'test-repo1', + url: 'https://gitlab.com/group1/test-repo1', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo1', + git_http_url: 'https://gitlab.com/group1/test-repo1.git', + git_ssh_url: 'git@gitlab.com:group1/test-repo1.git', + visibility_level: 20, + }, + }, +}; + +export const push_add_event_unmatched_group: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 6, + project: { + name: 'test-repo6', + path_with_namespace: 'awesome-group/test-repo6', + description: 'My Cool Project', + web_url: 'https://example.com/awesome-group/test-repo6', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/awesome-group/test-repo6', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: added_commits, + total_commits_count: 2, + repository: { + name: 'test-repo6', + url: 'https://gitlab.com/awesome-group/test-repo6', + description: 'My Cool Project', + homepage: 'https://gitlab.com/awesome-group/test-repo6', + git_http_url: 'https://gitlab.com/awesome-group/test-repo6.git', + git_ssh_url: 'git@gitlab.com:awesome-group/test-repo6.git', + visibility_level: 20, + }, + }, +}; +export const push_add_event_forked: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 4, + project: { + name: 'test-repo4-forked', + path_with_namespace: 'group1/test-repo4-forked', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo4-forked', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo4-forked', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: added_commits, + total_commits_count: 2, + repository: { + name: 'test-repo4-forked', + url: 'https://gitlab.com/group1/test-repo4-forked', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo4-forked', + git_http_url: 'https://gitlab.com/group1/test-repo4-forked.git', + git_ssh_url: 'git@gitlab.com:group1/test-repo4-forked.git', + visibility_level: 20, + }, + }, +}; + +export const push_remove_event: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 1, + project: { + name: 'test-repo', + path_with_namespace: 'group1/test-repo1', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo1', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo1', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: removed_commits, + total_commits_count: 2, + repository: { + name: 'test-repo', + url: 'https://gitlab.com/group1/test-repo1', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo1', + git_http_url: 'https://gitlab.com/group1/test-repo1.git', + git_ssh_url: 'git@gitlab.com:bgroup1/test-repo1.git', + visibility_level: 20, + }, + }, +}; + +export const push_modif_event: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 1, + project: { + name: 'test-repo', + path_with_namespace: 'group1/test-repo1', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo1', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo1', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: modified_commits, + total_commits_count: 2, + repository: { + name: 'test-repo', + url: 'https://gitlab.com/group1/test-repo1', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo1', + git_http_url: 'https://gitlab.com/group1/test-repo1.git', + git_ssh_url: 'git@gitlab.com:bgroup1/test-repo1.git', + visibility_level: 20, + }, + }, +}; + +/** + * Expected Backstage entities + */ +export const expected_location_entities: MockObject[] = + all_projects_response.map(project => { + const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${project.default_branch}/catalog-info.yaml`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ target: targetUrl, type: 'url' }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }); + +export const expected_added_location_entities: MockObject[] = added_commits.map( + commit => { + const targetUrl = `https://example.com/group1/test-repo1/-/blob/main/${commit.added}`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ target: targetUrl, type: 'url' }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }, +); + +export const expected_removed_location_entities: MockObject[] = + removed_commits.map(commit => { + const targetUrl = `https://example.com/group1/test-repo1/-/blob/main/${commit.removed}`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ target: targetUrl, type: 'url' }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }); + +export const expected_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group3', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group3', + 'example.com/team-path': 'group3', + }, + name: 'group3', + }, + spec: { + children: [], + profile: { + displayName: 'group3', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_group_user_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group1', + 'example.com/team-path': 'group1', + }, + name: 'group1', + }, + spec: { + children: [], + members: ['user1'], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_added_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/group-new/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group-new/group1', + 'example.com/team-path': 'group-new/group1', + }, + name: 'group-new-group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_removed_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/group-old/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group-old/group1', + 'example.com/team-path': 'group-old/group1', + }, + name: 'group-old-group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_transformed_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group3', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group3', + 'example.com/team-path': 'group3', + }, + name: '3', + }, + spec: { + children: [], + profile: { + displayName: 'group3', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_removed_user_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group3', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group3', + 'example.com/team-path': 'group3', + }, + name: '3', + }, + spec: { + children: [], + members: [], + profile: { + displayName: 'group3', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_single_user_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/JohnDoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/JohnDoe', + 'example.com/user-login': 'https://gitlab.example/john_doe', + }, + name: 'JohnDoe', + }, + spec: { + memberOf: [], + profile: { + displayName: 'John Doe', + email: 'john.doe@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_single_user_removed_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/johndoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/johndoe', + 'example.com/user-login': '', + }, + name: 'johndoe', + }, + spec: { + memberOf: [], + profile: { + displayName: 'John Doe', + email: 'john.doe@company.com', + picture: undefined, + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_full_org_scan_entities: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/JohnDoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/JohnDoe', + 'example.com/user-login': 'https://gitlab.example/john_doe', + }, + name: 'JohnDoe', + }, + spec: { + memberOf: ['group1'], + profile: { + displayName: 'John Doe', + email: 'john.doe@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/JaneDoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/JaneDoe', + 'example.com/user-login': 'https://gitlab.example/jane_doe', + }, + name: 'JaneDoe', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Jane Doe', + email: 'jane.doe@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/MarySmith', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/MarySmith', + 'example.com/user-login': 'https://gitlab.example/mary_smith', + }, + name: 'MarySmith', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Mary Smith', + email: 'mary.smith@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/MarioMario', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/MarioMario', + 'example.com/user-login': 'https://gitlab.example/mario_mario', + }, + name: 'MarioMario', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Mario Mario', + email: 'mario.mario-company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group1', + 'example.com/team-path': 'group1', + }, + name: 'group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_full_org_scan_entities_saas: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser1', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser1', + 'gitlab.com/user-login': 'https://gitlab.com/testuser1', + 'gitlab.com/saml-external-uid': '51', + }, + name: 'testuser1', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Test User 1', + email: 'testuser1@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser2', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser2', + 'gitlab.com/user-login': 'https://gitlab.com/testuser2', + 'gitlab.com/saml-external-uid': '52', + }, + name: 'testuser2', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Test User 2', + email: 'testuser2@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 99f6e7869b..9dfe8f8525 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -14,130 +14,25 @@ * limitations under the License. */ -import { ConfigReader } from '@backstage/config'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; -import { readGitLabIntegrationConfig } from '@backstage/integration'; import { getVoidLogger } from '@backstage/backend-common'; -import { graphql, rest } from 'msw'; -import { setupServer, SetupServer } from 'msw/node'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { readGitLabIntegrationConfig } from '@backstage/integration'; +import { setupServer } from 'msw/node'; +import { handlers } from '../__testUtils__/handlers'; +import * as mock from '../__testUtils__/mocks'; import { GitLabClient, paginated } from './client'; import { GitLabGroup, GitLabUser } from './types'; -const server = setupServer(); +const server = setupServer(...handlers); setupRequestMockHandlers(server); -const MOCK_CONFIG = readGitLabIntegrationConfig( - new ConfigReader({ - host: 'example.com', - token: 'test-token', - apiBaseUrl: 'https://example.com/api/v4', - baseUrl: 'https://example.com', - }), -); -const FAKE_PAGED_ENDPOINT = `/some-endpoint`; -const FAKE_PAGED_URL = `${MOCK_CONFIG.apiBaseUrl}${FAKE_PAGED_ENDPOINT}`; - -function setupFakeFourPageURL(srv: SetupServer, url: string) { - srv.use( - rest.get(url, (req, res, ctx) => { - const page = req.url.searchParams.get('page'); - const currentPage = page ? Number(page) : 1; - const fakePageCount = 4; - - return res( - // set next page number header if page requested is less than count - ctx.set( - 'x-next-page', - currentPage < fakePageCount ? String(currentPage + 1) : '', - ), - ctx.json([{ someContentOfPage: currentPage }]), - ); - }), - ); -} - -function setupFakeGroupProjectsEndpoint( - srv: SetupServer, - apiBaseUrl: string, - groupID: string, -) { - srv.use( - rest.get(`${apiBaseUrl}/groups/${groupID}/projects`, (_, res, ctx) => { - return res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - description: 'Project One Description', - name: 'Project One', - path: 'project-one', - }, - ]), - ); - }), - ); -} - -function setupFakeInstanceProjectsEndpoint( - srv: SetupServer, - apiBaseUrl: string, -) { - srv.use( - rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => { - return res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - description: 'Project One Description', - name: 'Project One', - path: 'project-one', - }, - { - id: 2, - description: 'Project Two Description', - name: 'Project Two', - path: 'project-two', - }, - ]), - ); - }), - ); -} - -function setupFakeHasFileEndpoint(srv: SetupServer, apiBaseUrl: string) { - srv.use( - rest.head( - `${apiBaseUrl}/projects/group%2Frepo/repository/files/catalog-info.yaml`, - (req, res, ctx) => { - const branch = req.url.searchParams.get('ref'); - if (branch === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); -} - -function setupFakeURLReturnEndpoint(srv: SetupServer, url: string) { - srv.use( - rest.get(url, (req, res, ctx) => { - return res(ctx.json([{ endpoint: req.url.toString() }])); - }), - ); -} - describe('GitLabClient', () => { describe('isSelfManaged', () => { it('returns true if self managed instance', () => { const client = new GitLabClient({ config: readGitLabIntegrationConfig( - new ConfigReader({ - host: 'example.com', - token: 'test-token', - apiBaseUrl: 'https://example.com/api/v4', - }), + new ConfigReader(mock.config_self_managed), ), logger: getVoidLogger(), }); @@ -145,12 +40,7 @@ describe('GitLabClient', () => { }); it('returns false if gitlab.com', () => { const client = new GitLabClient({ - config: readGitLabIntegrationConfig( - new ConfigReader({ - host: 'gitlab.com', - token: 'test-token', - }), - ), + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); expect(client.isSelfManaged()).toBeFalsy(); @@ -158,31 +48,30 @@ describe('GitLabClient', () => { }); describe('pagedRequest', () => { - beforeEach(() => { - // setup fake paginated endpoint with 4 pages each returning one item - setupFakeFourPageURL(server, FAKE_PAGED_URL); - }); - it('should provide immediate items within the page', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const { items } = await client.pagedRequest(FAKE_PAGED_ENDPOINT); + const { items } = await client.pagedRequest(mock.paged_endpoint); // fake page contains exactly one item expect(items).toHaveLength(1); }); it('should request items for a given page number', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const requestedPage = 2; const { items, nextPage } = await client.pagedRequest( - FAKE_PAGED_ENDPOINT, + mock.paged_endpoint, { page: requestedPage, }, @@ -195,12 +84,14 @@ describe('GitLabClient', () => { it('should not have a next page if at the end', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items, nextPage } = await client.pagedRequest( - FAKE_PAGED_ENDPOINT, + mock.paged_endpoint, { page: 4, }, @@ -211,50 +102,50 @@ describe('GitLabClient', () => { }); it('should throw if response is not okay', async () => { - const endpoint = '/unhealthy-endpoint'; - const url = `${MOCK_CONFIG.apiBaseUrl}${endpoint}`; - server.use( - rest.get(url, (_, res, ctx) => { - return res(ctx.status(400), ctx.json({ error: 'some error' })); - }), - ); - const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); // non-200 status code should throw - await expect(() => client.pagedRequest(endpoint)).rejects.toThrow(); + await expect(() => + client.pagedRequest(mock.unhealthy_endpoint), + ).rejects.toThrow(); }); }); + // TODO: review this. Make it so that the number of projects don't match the results describe('listProjects', () => { it('should get projects for a given group', async () => { - setupFakeGroupProjectsEndpoint( - server, - MOCK_CONFIG.apiBaseUrl, - 'test-group', - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const groupProjectsGen = paginated( options => client.listProjects(options), - { group: 'test-group' }, + { group: 1 }, ); + + const expectedProjects = mock.all_projects_response.filter(project => + project.path_with_namespace!.includes('group1/'), + ); + const allItems = []; for await (const item of groupProjectsGen) { allItems.push(item); } - expect(allItems).toHaveLength(1); + expect(allItems).toHaveLength(expectedProjects.length); }); it('should get all projects for an instance', async () => { - setupFakeInstanceProjectsEndpoint(server, MOCK_CONFIG.apiBaseUrl); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); @@ -266,532 +157,172 @@ describe('GitLabClient', () => { for await (const project of instanceProjects) { allProjects.push(project); } - expect(allProjects).toHaveLength(2); + expect(allProjects).toHaveLength(mock.all_projects_response.length); }); }); - it('listUsers gets all users in the instance', async () => { - server.use( - rest.get(`${MOCK_CONFIG.apiBaseUrl}/users`, (_, res, ctx) => - res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - username: 'test1', - name: 'Test Testit', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.example/test1', - created_at: '2023-01-19T07:27:03.333Z', - bio: '', - location: null, - public_email: null, - skype: '', - linkedin: '', - twitter: '', - website_url: '', - organization: null, - job_title: '', - pronouns: null, - bot: false, - work_information: null, - followers: 0, - following: 0, - is_followed: false, - local_time: null, - last_sign_in_at: '2023-01-19T07:27:49.601Z', - confirmed_at: '2023-01-19T07:27:02.905Z', - last_activity_on: '2023-01-19', - email: 'test@example.com', - theme_id: 1, - color_scheme_id: 1, - projects_limit: 100000, - current_sign_in_at: '2023-01-19T09:09:10.676Z', - identities: [], - can_create_group: true, - can_create_project: true, - two_factor_enabled: false, - external: false, - private_profile: false, - commit_email: 'test@example.com', - is_admin: false, - note: '', - }, - ]), + describe('listUsers', () => { + it('listUsers gets all users in the instance', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), ), - ), - ); - const client = new GitLabClient({ - config: MOCK_CONFIG, - logger: getVoidLogger(), + logger: getVoidLogger(), + }); + + const allUsers: GitLabUser[] = []; + for await (const user of paginated( + options => client.listUsers(options), + {}, + )) { + allUsers.push(user); + } + + expect(allUsers).toMatchObject(mock.all_users_response); }); - - const allUsers: GitLabUser[] = []; - for await (const user of paginated( - options => client.listUsers(options), - {}, - )) { - allUsers.push(user); - } - - expect(allUsers).toMatchObject([ - { - id: 1, - username: 'test1', - email: 'test@example.com', - name: 'Test Testit', - state: 'active', - web_url: 'https://gitlab.example/test1', - avatar_url: 'https://secure.gravatar.com/', - }, - ]); }); - it('listGroups gets all groups in the instance', async () => { - server.use( - rest.get(`${MOCK_CONFIG.apiBaseUrl}/groups`, (_, res, ctx) => - res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - web_url: 'https://gitlab.example/groups/group1', - name: 'group1', - path: 'group1', - description: '', - visibility: 'internal', - share_with_group_lock: false, - require_two_factor_authentication: false, - two_factor_grace_period: 48, - project_creation_level: 'developer', - auto_devops_enabled: null, - subgroup_creation_level: 'owner', - emails_disabled: null, - mentions_disabled: null, - lfs_enabled: true, - default_branch_protection: 2, - avatar_url: null, - request_access_enabled: false, - full_name: '8020', - full_path: '8020', - created_at: '2017-06-19T06:42:34.160Z', - parent_id: null, - }, - ]), + describe('listGroups', () => { + it('listGroups gets all groups in the instance', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), ), - ), - ); - const client = new GitLabClient({ - config: MOCK_CONFIG, - logger: getVoidLogger(), + logger: getVoidLogger(), + }); + + const allGroups: GitLabGroup[] = []; + for await (const group of paginated( + options => client.listGroups(options), + {}, + )) { + allGroups.push(group); + } + + expect(allGroups).toMatchObject(mock.all_groups_response); }); - - const allGroups: GitLabGroup[] = []; - for await (const group of paginated( - options => client.listGroups(options), - {}, - )) { - allGroups.push(group); - } - - expect(allGroups).toMatchObject([ - { - id: 1, - name: 'group1', - full_path: '8020', - description: '', - parent_id: null, - }, - ]); }); describe('get gitlab.com users', () => { it('gets all users under group', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: [ - { - user: { - id: 'gid://gitlab/User/1', - username: 'user1', - publicEmail: 'user1@example.com', - name: 'user1', - state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); - const saasMembers = ( - await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + await client.getGroupMembers('saas-multi-user-group', [ + 'DIRECT, DESCENDANTS', + ]) ).items; - const expectedSaasMember = [ - { - id: 1, - username: 'user1', - email: 'user1@example.com', - name: 'user1', - state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', - }, - ]; - expect(saasMembers.length).toEqual(1); - expect(saasMembers).toEqual(expectedSaasMember); + expect(saasMembers.length).toEqual(2); + expect(saasMembers).toEqual(mock.expectedSaasMember); }); - it('gets all users with token without full permissions', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: {}, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); - const saasMembers = ( - await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + await client.getGroupMembers('', ['DIRECT, DESCENDANTS']) ).items; - expect(saasMembers).toEqual([]); }); - it('rejects when GraphQL returns errors', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.errors([ - { message: 'Unexpected end of document', locations: [] }, - ]), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); - await expect(() => - client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']), + client.getGroupMembers('error-group', ['DIRECT, DESCENDANTS']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); it('traverses multi-page results', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: req.variables.endCursor - ? [ - { - user: { - id: 'gid://gitlab/User/1', - username: 'user1', - publicEmail: 'user1@example.com', - name: 'user1', - state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', - }, - }, - ] - : [ - { - user: { - id: 'gid://gitlab/User/2', - username: 'user2', - publicEmail: 'user2@example.com', - name: 'user2', - state: 'active', - webUrl: 'user2.com', - avatarUrl: 'user2', - }, - }, - ], - pageInfo: { - endCursor: req.variables.endCursor ? 'end' : 'next', - hasNextPage: !req.variables.endCursor, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const saasMembers = ( - await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + await client.getGroupMembers('multi-page-saas', ['DIRECT, DESCENDANTS']) ).items; - const expectedSaasMember1 = { - id: 1, - username: 'user1', - email: 'user1@example.com', - name: 'user1', - state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', - }; - - const expectedSaasMember2 = { - id: 2, - username: 'user2', - email: 'user2@example.com', - name: 'user2', - state: 'active', - web_url: 'user2.com', - avatar_url: 'user2', - }; - expect(saasMembers.length).toEqual(2); - expect(saasMembers[0]).toEqual(expectedSaasMember2); - expect(saasMembers[1]).toEqual(expectedSaasMember1); + expect(saasMembers[0]).toEqual(mock.expectedSaasMember[1]); + expect(saasMembers[1]).toEqual(mock.expectedSaasMember[0]); }); }); describe('listDescendantGroups', () => { it('gets all groups under root', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.data({ - group: { - descendantGroups: { - nodes: [ - { - id: 'gid://gitlab/Group/1', - name: 'group1', - description: 'description1', - fullPath: 'path/group1', - parent: { - id: '123', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const saasGroups = (await client.listDescendantGroups('group1')).items; + const allGroups = (await client.listDescendantGroups('group-with-parent')) + .items; - const expectedSaasGroup = [ - { - id: 1, - name: 'group1', - description: 'description1', - full_path: 'path/group1', - parent_id: 123, - }, - ]; - - expect(saasGroups.length).toEqual(1); - expect(saasGroups).toEqual(expectedSaasGroup); + expect(allGroups.length).toEqual(1); + expect(allGroups).toEqual(mock.group_with_parent); }); it('gets all descendant groups with token without full permissions', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.data({ - group: {}, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const saasGroups = (await client.listDescendantGroups('group1')).items; + const allGroups = ( + await client.listDescendantGroups('non-existing-group') + ).items; - expect(saasGroups).toEqual([]); + expect(allGroups).toEqual([]); }); it('rejects when GraphQL returns errors', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.errors([ - { message: 'Unexpected end of document', locations: [] }, - ]), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - await expect(() => client.listDescendantGroups('group1')).rejects.toThrow( + await expect(() => + client.listDescendantGroups('error-group'), + ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); it('traverses multi-page results', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (req, res, ctx) => - res( - ctx.data({ - group: { - descendantGroups: { - nodes: req.variables.endCursor - ? [ - { - id: 'gid://gitlab/Group/1', - name: 'group1', - description: 'description1', - fullPath: 'path/group1', - parent: { - id: '123', - }, - }, - ] - : [ - { - id: 'gid://gitlab/Group/2', - name: 'group2', - description: 'description2', - fullPath: 'path/group2', - parent: { - id: '123', - }, - }, - ], - pageInfo: { - endCursor: req.variables.endCursor ? 'end' : 'next', - hasNextPage: !req.variables.endCursor, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const saasGroups = (await client.listDescendantGroups('root')).items; - const expectedSaasGroup1 = { - id: 1, - name: 'group1', - description: 'description1', - full_path: 'path/group1', - parent_id: 123, - }; - - const expectedSaasGroup2 = { - id: 2, - name: 'group2', - description: 'description2', - full_path: 'path/group2', - parent_id: 123, - }; - expect(saasGroups.length).toEqual(2); - expect(saasGroups[0]).toEqual(expectedSaasGroup2); - expect(saasGroups[1]).toEqual(expectedSaasGroup1); + expect(saasGroups[0]).toEqual(mock.expectedSaasGroup[1]); + expect(saasGroups[1]).toEqual(mock.expectedSaasGroup[0]); }); }); describe('getGroupMembers', () => { it('gets member IDs', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: [ - { - user: { - id: 'gid://gitlab/User/1', - username: 'user1', - publicEmail: 'user1@example.com', - name: 'user1', - state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); @@ -811,96 +342,144 @@ describe('GitLabClient', () => { }); it('gets member IDs with token without full permissions', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: {}, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', ['DIRECT']); + const members = await client.getGroupMembers('non-existing-group', [ + 'DIRECT', + ]); expect(members.items).toEqual([]); }); + // TODO: is this one really necessary? it('rejects when GraphQL returns errors', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.errors([ - { message: 'Unexpected end of document', locations: [] }, - ]), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); await expect(() => - client.getGroupMembers('group1', ['DIRECT']), + client.getGroupMembers('error-group', ['DIRECT']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); it('traverses multi-page results', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: req.variables.endCursor - ? [{ user: { id: 'gid://gitlab/User/2' } }] - : [{ user: { id: 'gid://gitlab/User/1' } }], - pageInfo: { - endCursor: req.variables.endCursor ? 'end' : 'next', - hasNextPage: !req.variables.endCursor, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', ['DIRECT']); + const members = await client.getGroupMembers('multi-page', ['DIRECT']); expect(members.items[0].id).toEqual(1); expect(members.items[1].id).toEqual(2); }); }); + + describe('getGroupById', () => { + it('should return group details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + const group = await client.getGroupById(1); + expect(group).toMatchObject(mock.all_groups_response[0]); + }); + + it('should handle errors when fetching group details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + await expect(() => client.getGroupById(42)).rejects.toThrow( + 'Internal Server Error', + ); + }); + }); + + describe('getProjectById', () => { + it('should return project details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + const project = await client.getProjectById(1); + expect(project).toMatchObject(mock.all_projects_response[0]); + }); + + it('should handle errors when fetching project details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + await expect(() => client.getProjectById(42)).rejects.toThrow( + 'Internal Server Error', + ); + }); + }); + + describe('getUserById', () => { + it('should return user details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + const user = await client.getUserById(1); + expect(user).toMatchObject(mock.all_users_response[0]); + }); + + it('should handle errors when fetching user details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + await expect(() => client.getUserById(42)).rejects.toThrow( + 'Internal Server Error', + ); + }); + }); }); describe('paginated', () => { it('should iterate through the pages until exhausted', async () => { - setupFakeFourPageURL(server, FAKE_PAGED_URL); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const paginatedItems = paginated( - options => client.pagedRequest(FAKE_PAGED_ENDPOINT, options), + options => client.pagedRequest(mock.paged_endpoint, options), {}, ); const allItems = []; @@ -916,25 +495,26 @@ describe('hasFile', () => { let client: GitLabClient; beforeEach(() => { - setupFakeHasFileEndpoint(server, MOCK_CONFIG.apiBaseUrl); client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); }); - it('should not find catalog file', async () => { + it('should find catalog file', async () => { const hasFile = await client.hasFile( - 'group/repo', - 'master', + 'group1/test-repo1', + 'main', 'catalog-info.yaml', ); expect(hasFile).toBe(true); }); - it('should find catalog file', async () => { + it('should not find catalog file', async () => { const hasFile = await client.hasFile( - 'group/repo', + 'group1/test-repo1', 'unknown', 'catalog-info.yaml', ); @@ -943,19 +523,16 @@ describe('hasFile', () => { }); describe('pagedRequest search params', () => { - beforeEach(() => { - // setup fake paginated endpoint with 4 pages each returning one item - setupFakeURLReturnEndpoint(server, FAKE_PAGED_URL); - }); - it('no search params provided', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, ); // fake page contains exactly one item expect(items).toHaveLength(1); @@ -966,12 +543,14 @@ describe('pagedRequest search params', () => { it('defined numeric search params', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, { page: 1, per_page: 50 }, ); // fake page contains exactly one item @@ -985,12 +564,14 @@ describe('pagedRequest search params', () => { it('defined string search params', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, { test: 'value', empty: '' }, ); // fake page contains exactly one item @@ -1004,12 +585,14 @@ describe('pagedRequest search params', () => { it('defined boolean search params', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, { active: true, archived: false }, ); // fake page contains exactly one item diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index cc2b2362cd..8f56657f68 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -24,6 +24,7 @@ import { GitLabDescendantGroupsResponse, GitLabGroup, GitLabGroupMembersResponse, + GitLabProject, GitLabUser, PagedResponse, } from './types'; @@ -80,6 +81,39 @@ export class GitLabClient { return this.pagedRequest(`/projects`, options); } + async getProjectById( + projectId: number, + options?: CommonListOptions, + ): Promise { + // Make the request to the GitLab API + const response = await this.nonPagedRequest( + `/projects/${projectId}`, + options, + ); + + return response; + } + + async getGroupById( + groupId: number, + options?: CommonListOptions, + ): Promise { + // Make the request to the GitLab API + const response = await this.nonPagedRequest(`/groups/${groupId}`, options); + + return response; + } + + async getUserById( + userId: number, + options?: CommonListOptions, + ): Promise { + // Make the request to the GitLab API + const response = await this.nonPagedRequest(`/users/${userId}`, options); + + return response; + } + async listUsers( options?: UserListOptions, ): Promise> { @@ -325,9 +359,13 @@ export class GitLabClient { options?: CommonListOptions, ): Promise> { const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); + for (const key in options) { - if (options[key] !== undefined && options[key] !== '') { - request.searchParams.append(key, options[key]!.toString()); + if (options.hasOwnProperty(key)) { + const value = options[key]; + if (value !== undefined && value !== '') { + request.searchParams.append(key, value.toString()); + } } } @@ -336,6 +374,7 @@ export class GitLabClient { request.toString(), getGitLabRequestOptions(this.config), ); + if (!response.ok) { throw new Error( `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ @@ -343,6 +382,7 @@ export class GitLabClient { } - ${response.statusText}`, ); } + return response.json().then(items => { const nextPage = response.headers.get('x-next-page'); @@ -352,6 +392,37 @@ export class GitLabClient { } as PagedResponse; }); } + + async nonPagedRequest( + endpoint: string, + options?: CommonListOptions, + ): Promise { + const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); + + for (const key in options) { + if (options.hasOwnProperty(key)) { + const value = options[key]; + if (value !== undefined && value !== '') { + request.searchParams.append(key, value.toString()); + } + } + } + + const response = await fetch( + request.toString(), + getGitLabRequestOptions(this.config), + ); + + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + + return response.json(); + } } /** diff --git a/plugins/catalog-backend-module-gitlab/src/lib/index.ts b/plugins/catalog-backend-module-gitlab/src/lib/index.ts index 1eee268862..cc82bd6a0b 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/index.ts @@ -14,14 +14,15 @@ * limitations under the License. */ +export { readGitlabConfigs } from '../providers/config'; export { GitLabClient, paginated } from './client'; export type { - GitLabUser, GitLabGroup, GitLabGroupSamlIdentity, GitLabProject, - GitlabProviderConfig, + GitLabUser, GitlabGroupDescription, + GitlabProviderConfig, GroupNameTransformer, GroupNameTransformerOptions, GroupTransformer, @@ -29,4 +30,3 @@ export type { UserTransformer, UserTransformerOptions, } from './types'; -export { readGitlabConfigs } from '../providers/config'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 123515ccf3..2857fea162 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -34,6 +34,9 @@ export type GitlabProjectForkedFrom = { export type GitLabProject = { id: number; + description?: string; + name?: string; + path?: string; default_branch?: string; archived: boolean; last_activity_at: string; @@ -180,6 +183,11 @@ export type GitlabProviderConfig = { */ groupPattern: RegExp; + /** + * If true, the provider will also ingest add inherited users to the ingested groups + */ + allowInherited?: boolean; + orgEnabled?: boolean; schedule?: TaskScheduleDefinition; /** @@ -242,3 +250,72 @@ export interface GroupTransformerOptions { providerConfig: GitlabProviderConfig; groupNameTransformer: GroupNameTransformer; } + +/** + * Represents the schema for system hook events related to groups. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseGroupEventsSchema = { + created_at: string; + updated_at: string; + name: string; + path: string; + full_path: string; + group_id: number; +}; + +/** + * Represents the schema for system hook events related to users. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseUserEventsSchema = { + created_at: string; + updated_at: string; + email: string; + name: string; + username: string; + user_id: number; +}; + +/** + * Represents the schema for system hook events related to user memberships. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseMembershipEventsSchema = { + created_at: string; + updated_at: string; + group_name: string; + group_path: string; + group_id: number; + user_username: string; + user_email: string; + user_name: string; + user_id: number; + group_access: string; +}; + +/** + * Represents the schema for system hook events related to projects. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseProjectEventsSchema = { + created_at: string; + updated_at: string; + event_name: string; + name: string; + owner_email: string; + owner_name: string; + owners: { name: string; email: string }[]; + path: string; + path_with_namespace: string; + project_id: number; + project_visibility: string; +}; diff --git a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts index 46cfe403e8..3c93cb98f1 100644 --- a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts @@ -14,15 +14,27 @@ * limitations under the License. */ +import { createServiceFactory } from '@backstage/backend-plugin-api'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { TestEventsService } from '@backstage/plugin-events-backend-test-utils'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; import { Duration } from 'luxon'; -import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider'; import { GitlabDiscoveryEntityProvider } from '../providers'; +import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider'; describe('catalogModuleGitlabDiscoveryEntityProvider', () => { it('should register provider at the catalog extension point', async () => { + const events = new TestEventsService(); + const eventsServiceFactory = createServiceFactory({ + service: eventsServiceRef, + deps: {}, + async factory({}) { + return events; + }, + }); let addedProviders: Array | undefined; let usedSchedule: TaskScheduleDefinition | undefined; @@ -31,6 +43,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => { addedProviders = providers; }, }; + const connection = jest.fn() as unknown as EntityProviderConnection; const runner = jest.fn(); const scheduler = mockServices.scheduler.mock({ createScheduledTaskRunner(schedule) { @@ -68,6 +81,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => { await startTestBackend({ extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], features: [ + eventsServiceFactory(), catalogModuleGitlabDiscoveryEntityProvider(), mockServices.rootConfig.factory({ data: config }), mockServices.logger.factory(), @@ -78,9 +92,17 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => { expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); expect(addedProviders?.length).toEqual(1); - expect(addedProviders?.pop()?.getProviderName()).toEqual( + expect(runner).not.toHaveBeenCalled(); + + const provider = addedProviders!.pop()!; + expect(provider.getProviderName()).toEqual( 'GitlabDiscoveryEntityProvider:test-id', ); - expect(runner).not.toHaveBeenCalled(); + await provider.connect(connection); + expect(events.subscribed).toHaveLength(1); + expect(events.subscribed[0].id).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + expect(runner).toHaveBeenCalledTimes(1); }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts index a1b24e4eea..bba8348a74 100644 --- a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts @@ -19,6 +19,7 @@ import { createBackendModule, } from '@backstage/backend-plugin-api'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; import { GitlabDiscoveryEntityProvider } from '../providers'; /** @@ -26,6 +27,7 @@ import { GitlabDiscoveryEntityProvider } from '../providers'; * * @alpha */ + export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({ pluginId: 'catalog', moduleId: 'gitlab-discovery-entity-provider', @@ -36,14 +38,16 @@ export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({ catalog: catalogProcessingExtensionPoint, logger: coreServices.logger, scheduler: coreServices.scheduler, + events: eventsServiceRef, }, - async init({ config, catalog, logger, scheduler }) { - catalog.addEntityProvider( + async init({ config, catalog, logger, scheduler, events }) { + const gitlabDiscoveryEntityProvider = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, + events, scheduler, - }), - ); + }); + catalog.addEntityProvider(gitlabDiscoveryEntityProvider); }, }); }, diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index 7508b35b84..7167fc1c2d 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -23,10 +23,16 @@ import { import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; -import { rest } from 'msw'; +import { DefaultEventsService } from '@backstage/plugin-events-node'; import { setupServer } from 'msw/node'; +import { handlers } from '../__testUtils__/handlers'; +import * as mock from '../__testUtils__/mocks'; import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; +const server = setupServer(...handlers); +setupRequestMockHandlers(server); +afterEach(() => jest.resetAllMocks()); + class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -42,13 +48,8 @@ class PersistingTaskRunner implements TaskRunner { const logger = getVoidLogger(); -const server = setupServer(); - -describe('GitlabDiscoveryEntityProvider', () => { - setupRequestMockHandlers(server); - afterEach(() => jest.resetAllMocks()); - - it('no provider config', () => { +describe('GitlabDiscoveryEntityProvider - configuration', () => { + it('should not instantiate providers when no config found', () => { const schedule = new PersistingTaskRunner(); const config = new ConfigReader({}); const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { @@ -59,29 +60,46 @@ describe('GitlabDiscoveryEntityProvider', () => { expect(providers).toHaveLength(0); }); - it('single simple discovery config', () => { + it('should fail without schedule nor scheduler', () => { + const config = new ConfigReader(mock.config_single_integration_branch); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('should fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader(mock.config_no_schedule_integration); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id', + ); + }); + it('should throw error when no matching GitLab integration config found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_github_host); + + expect(() => { + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow('No gitlab integration found that matches host example.com'); + }); + + it('should instantiate provider with single simple discovery config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader(mock.config_single_integration_branch); const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -93,33 +111,9 @@ describe('GitlabDiscoveryEntityProvider', () => { ); }); - it('multiple discovery configs', () => { + it('should instantiate providers when multiple discovery configs', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - 'second-test': { - host: 'test-gitlab', - group: 'second-group', - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_double_integration); const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -133,29 +127,10 @@ describe('GitlabDiscoveryEntityProvider', () => { 'GitlabDiscoveryEntityProvider:second-test', ); }); - - it('apply full update on scheduled execution', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); +}); +describe('GitlabDiscoveryEntityProvider - refresh', () => { + it('should apply full update on scheduled execution', async () => { + const config = new ConfigReader(mock.config_no_org_integration); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -169,92 +144,28 @@ describe('GitlabDiscoveryEntityProvider', () => { 'GitlabDiscoveryEntityProvider:test-id', ); - server.use( - rest.get( - `https://api.gitlab.example/api/v4/groups/test-group/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); - await provider.connect(entityProviderConnection); const taskDef = schedule.getTasks()[0]; expect(taskDef.id).toEqual('GitlabDiscoveryEntityProvider:test-id:refresh'); await (taskDef.fn as () => Promise)(); - const url = `https://api.gitlab.example/test-group/test-repo/-/blob/master/catalog-info.yaml`; - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': `url:${url}`, - 'backstage.io/managed-by-origin-location': `url:${url}`, - }, - name: 'generated-cd37bf72a2fe92603f4255d9f49c6c1ead746a48', - }, - spec: { - presence: 'optional', - target: `${url}`, - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ]; - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: expectedEntities, + entities: mock.expected_location_entities.filter( + entity => + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), }); }); it('should filter found projects based on a provided project pattern', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - projectPattern: 'john/', - }, - }, - }, - }, - }); + const config = new ConfigReader( + mock.config_single_integration_project_pattern, + ); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -265,50 +176,10 @@ describe('GitlabDiscoveryEntityProvider', () => { schedule, })[0]; - server.use( - rest.get( - `https://api.gitlab.example/api/v4/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - { - id: 124, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/john/example', - path_with_namespace: 'john/example', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); + const projectPattern = + mock.config_single_integration_project_pattern.catalog.providers.gitlab[ + 'test-id' + ].projectPattern; await provider.connect(entityProviderConnection); @@ -316,55 +187,20 @@ describe('GitlabDiscoveryEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - 'backstage.io/managed-by-origin-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - }, - name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9', - }, - spec: { - presence: 'optional', - target: - 'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ], + entities: mock.expected_location_entities.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(projectPattern) && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), }); }); it('should filter fork projects', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - skipForkedRepos: true, - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_single_integration_skip_forks); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -375,216 +211,26 @@ describe('GitlabDiscoveryEntityProvider', () => { schedule, })[0]; - server.use( - rest.get( - `https://api.gitlab.example/api/v4/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - forked_from_project: { - id: 13083, - }, - }, - { - id: 124, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/john/example', - path_with_namespace: 'john/example', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); - await provider.connect(entityProviderConnection); await provider.refresh(logger); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - 'backstage.io/managed-by-origin-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - }, - name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9', - }, - spec: { - presence: 'optional', - target: - 'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ], + entities: mock.expected_location_entities.filter( + entity => + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('forked') && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), }); }); - it('fail without schedule and scheduler', () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); - - expect(() => - GitlabDiscoveryEntityProvider.fromConfig(config, { - logger, - }), - ).toThrow('Either schedule or scheduler must be provided'); - }); - - it('fail with scheduler but no schedule config', () => { - const scheduler = { - createScheduledTaskRunner: (_: any) => jest.fn(), - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); - - expect(() => - GitlabDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ).toThrow( - 'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id', - ); - }); - - it('single simple provider config with schedule in config', async () => { - const schedule = new PersistingTaskRunner(); - const scheduler = { - createScheduledTaskRunner: (_: any) => schedule, - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - schedule: { - frequency: 'PT30M', - timeout: 'PT3M', - }, - }, - }, - }, - }, - }); - const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }); - - expect(providers).toHaveLength(1); - expect(providers[0].getProviderName()).toEqual( - 'GitlabDiscoveryEntityProvider:test-id', - ); - }); - it('should filter found projects based on the branch', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - branch: 'test', - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_single_integration_branch); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -595,47 +241,9 @@ describe('GitlabDiscoveryEntityProvider', () => { schedule, })[0]; - server.use( - rest.get( - `https://api.gitlab.example/api/v4/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - { - id: 124, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/john/example', - path_with_namespace: 'john/example', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (_, res, ctx) => { - return res(ctx.status(404, 'Not Found')); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'test') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); + const configured_branch = + mock.config_single_integration_branch.catalog.providers.gitlab['test-id'] + .branch; await provider.connect(entityProviderConnection); @@ -643,30 +251,229 @@ describe('GitlabDiscoveryEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml', - 'backstage.io/managed-by-origin-location': - 'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml', - }, - name: 'generated-232185d858fee049986d202c10316d634e76a3d1', - }, - spec: { - presence: 'optional', - target: - 'https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml', - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ], + entities: mock.expected_location_entities.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(configured_branch) && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), + }); + }); + + it('should only include projects with fallback branch', async () => { + const config = new ConfigReader(mock.config_fallbackBranch_branch); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const configured_branch = + mock.config_fallbackBranch_branch.catalog.providers.gitlab['test-id'] + .fallbackBranch; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(configured_branch) && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), + }); + }); + + it('should ignore projects outside group scope', async () => { + const config = new ConfigReader(mock.config_single_integration_group); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const configured_group = + mock.config_single_integration_group.catalog.providers.gitlab['test-id'] + .group; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities.filter(entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(configured_group), + ), }); }); }); +describe('GitlabDiscoveryEntityProvider - events', () => { + it('should ignore push event if project is forked', async () => { + const config = new ConfigReader(mock.config_single_integration_skip_forks); + + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event_forked); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it(`should skip refresh and mutation when project pattern doesn't match`, async () => { + const config = new ConfigReader(mock.config_unmatched_project_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should ignore projects outside group scope', async () => { + const config = new ConfigReader(mock.config_single_integration_group); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event_unmatched_group); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + it('should apply delta mutations on added files from push event', async () => { + const config = new ConfigReader(mock.config_single_integration_branch); + + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_added_location_entities, + removed: [], + }); + }); + + it('should apply delta mutations on removed files from push event', async () => { + const config = new ConfigReader(mock.config_single_integration_branch); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_remove_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: mock.expected_removed_location_entities, + }); + }); + + it('should call refresh on added files from push event', async () => { + const config = new ConfigReader(mock.config_single_integration_branch); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + const url = `https://example.com/group1/test-repo1`; + + await events.publish(mock.push_modif_event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.refresh).toHaveBeenCalledWith({ + keys: [ + `url:${url}/-/tree/main/catalog-info.yaml`, + `url:${url}/-/blob/main/catalog-info.yaml`, + ], + }); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + // EventSupportChange: stop add tests >>> +}); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index ae8fe4e12c..73ff5f3a01 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -17,12 +17,15 @@ import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, - LocationSpec, locationSpecToLocationEntity, } from '@backstage/plugin-catalog-node'; +import { EventsService } from '@backstage/plugin-events-node'; +import { WebhookProjectSchema, WebhookPushEventSchema } from '@gitbeaker/rest'; import * as uuid from 'uuid'; import { GitLabClient, @@ -33,26 +36,38 @@ import { } from '../lib'; import { LoggerService } from '@backstage/backend-plugin-api'; +import * as path from 'path'; + +const TOPIC_REPO_PUSH = 'gitlab.push'; + type Result = { scanned: number; matches: GitLabProject[]; }; /** - * Discovers entity definition files in the groups of a Gitlab instance. + * Discovers catalog files located in your GitLab instance. + * The provider will search your GitLab instance's projects and register catalog files matching the configured path + * as Location entity and via following processing steps add all contained catalog entities. + * This can be useful as an alternative to static locations or manually adding things to the catalog. + * * @public */ +// <<< EventSupportChange: implemented EventSubscriber interface export class GitlabDiscoveryEntityProvider implements EntityProvider { private readonly config: GitlabProviderConfig; private readonly integration: GitLabIntegration; private readonly logger: LoggerService; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; + private readonly events?: EventsService; + private readonly gitLabClient: GitLabClient; static fromConfig( config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; }, @@ -92,13 +107,20 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }), ); }); + return providers; } + /** + * Constructs a GitlabDiscoveryEntityProvider instance. + * + * @param options - Configuration options including config, integration, logger, and taskRunner. + */ private constructor(options: { config: GitlabProviderConfig; integration: GitLabIntegration; logger: LoggerService; + events?: EventsService; taskRunner: TaskRunner; }) { this.config = options.config; @@ -107,6 +129,11 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { target: this.getProviderName(), }); this.scheduleFn = this.createScheduleFn(options.taskRunner); + this.events = options.events; + this.gitLabClient = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); } getProviderName(): string { @@ -116,8 +143,28 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { async connect(connection: EntityProviderConnection): Promise { this.connection = connection; await this.scheduleFn(); + + if (this.events) { + await this.events.subscribe({ + id: this.getProviderName(), + topics: [TOPIC_REPO_PUSH], + onEvent: async params => { + if (params.topic !== TOPIC_REPO_PUSH) { + return; + } + + await this.onRepoPush(params.eventPayload as WebhookPushEventSchema); + }, + }); + } } + /** + * Creates a scheduled task runner for refreshing the entity provider. + * + * @param taskRunner - The task runner instance. + * @returns The scheduled function. + */ private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; @@ -143,6 +190,11 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }; } + /** + * Performs a full scan on the GitLab instance searching for locations to be ingested + * + * @param logger - The logger instance for logging. + */ async refresh(logger: LoggerService): Promise { if (!this.connection) { throw new Error( @@ -150,13 +202,8 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { ); } - const client = new GitLabClient({ - config: this.integration.config, - logger: logger, - }); - const projects = paginated( - options => client.listProjects(options), + options => this.gitLabClient.listProjects(options), { archived: false, group: this.config.group, @@ -171,43 +218,17 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }; for await (const project of projects) { - if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) { - continue; - } - - res.scanned++; - - if ( - this.config.skipForkedRepos && - project.hasOwnProperty('forked_from_project') - ) { - continue; - } - - if ( - !this.config.branch && - this.config.fallbackBranch === '*' && - project.default_branch === undefined - ) { - continue; - } - - const project_branch = - this.config.branch ?? - project.default_branch ?? - this.config.fallbackBranch; - - const projectHasFile: boolean = await client.hasFile( - project.path_with_namespace ?? '', - project_branch, - this.config.catalogFile, - ); - if (projectHasFile) { + if (await this.shouldProcessProject(project, this.gitLabClient)) { + res.scanned++; res.matches.push(project); } } const locations = res.matches.map(p => this.createLocationSpec(p)); + + logger.info( + `Processed ${locations.length} from scanned ${res.scanned} projects.`, + ); await this.connection.applyMutation({ type: 'full', entities: locations.map(location => ({ @@ -222,10 +243,250 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { this.config.branch ?? project.default_branch ?? this.config.fallbackBranch; + return { type: 'url', target: `${project.web_url}/-/blob/${project_branch}/${this.config.catalogFile}`, presence: 'optional', }; } + + /** + * Handles the "gitlab.push" event. + * + * @param event - The push event payload. + */ + private async onRepoPush(event: WebhookPushEventSchema): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + this.logger.info( + `Received push event for ${event.project.path_with_namespace}`, + ); + + const project = await this.gitLabClient.getProjectById(event.project_id); + + if (!project) { + this.logger.debug( + `Ignoring push event for ${event.project.path_with_namespace}`, + ); + + return; + } + + if (!(await this.shouldProcessProject(project, this.gitLabClient))) { + this.logger.debug(`Skipping event ${event.project.path_with_namespace}`); + return; + } + + // Get array of added, removed or modified files from the push event + const added = this.getFilesMatchingConfig( + event, + 'added', + this.config.catalogFile, + ); + const removed = this.getFilesMatchingConfig( + event, + 'removed', + this.config.catalogFile, + ); + const modified = this.getFilesMatchingConfig( + event, + 'modified', + this.config.catalogFile, + ); + + // Modified files will be scheduled to a refresh + const addedEntities = this.createLocationSpecCommitedFiles( + event.project, + added, + ); + + const removedEntities = this.createLocationSpecCommitedFiles( + event.project, + removed, + ); + + if (addedEntities.length > 0 || removedEntities.length > 0) { + await this.connection.applyMutation({ + type: 'delta', + added: this.toDeferredEntities( + addedEntities.map(entity => entity.target), + ), + removed: this.toDeferredEntities( + removedEntities.map(entity => entity.target), + ), + }); + } + + if (modified.length > 0) { + const projectBranch = + this.config.branch ?? + event.project.default_branch ?? + this.config.fallbackBranch; + + // scheduling a refresh to both tree and blob (https://git-scm.com/book/en/v2/Git-Internals-Git-Objects) + await this.connection.refresh({ + keys: [ + ...modified.map( + filePath => + `url:${event.project.web_url}/-/tree/${projectBranch}/${filePath}`, + ), + ...modified.map( + filePath => + `url:${event.project.web_url}/-/blob/${projectBranch}/${filePath}`, + ), + ], + }); + } + + this.logger.info( + `Processed GitLab push event from ${event.project.web_url}: added ${added.length} - removed ${removed.length} - modified ${modified.length}`, + ); + } + + /** + * Gets files matching the specified commit action and catalog file name. + * + * @param event - The push event payload. + * @param action - The action type ('added', 'removed', or 'modified'). + * @param catalogFile - The catalog file name. + * @returns An array of file paths. + */ + private getFilesMatchingConfig( + event: WebhookPushEventSchema, + action: 'added' | 'removed' | 'modified', + catalogFile: string, + ): string[] { + if (!event.commits) { + return []; + } + + const matchingFiles = event.commits.flatMap((element: any) => + element[action].filter( + (file: string) => path.basename(file) === catalogFile, + ), + ); + + if (matchingFiles.length === 0) { + this.logger.debug( + `No files matching '${catalogFile}' found in the commits.`, + ); + } + + return matchingFiles; + } + + /** + * Creates Backstage location specs for committed files. + * + * @param project - The GitLab project information. + * @param addedFiles - The array of added file paths. + * @returns An array of location specs. + */ + private createLocationSpecCommitedFiles( + project: WebhookProjectSchema, + addedFiles: string[], + ): LocationSpec[] { + const projectBranch = + this.config.branch ?? + project.default_branch ?? + this.config.fallbackBranch; + + // Filter added files that match the catalog file pattern + const matchingFiles = addedFiles.filter( + file => path.basename(file) === this.config.catalogFile, + ); + + // Create a location spec for each matching file + const locationSpecs: LocationSpec[] = matchingFiles.map(file => ({ + type: 'url', + target: `${project.web_url}/-/blob/${projectBranch}/${file}`, + presence: 'optional', + })); + + return locationSpecs; + } + + /** + * Converts a target URL to a LocationSpec object. + * + * @param {string} target - The target URL to be converted. + * @returns {LocationSpec} The LocationSpec object representing the URL. + */ + private toLocationSpec(target: string): LocationSpec { + return { + type: 'url', + target: target, + presence: 'optional', + }; + } + + private toDeferredEntities(targets: string[]): DeferredEntity[] { + return targets + .map(target => { + const location = this.toLocationSpec(target); + + return locationSpecToLocationEntity({ location }); + }) + .map(entity => { + return { + locationKey: this.getProviderName(), + entity: entity, + }; + }); + } + + private async shouldProcessProject( + project: GitLabProject, + client: GitLabClient, + ): Promise { + if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it does not match the project pattern ${this.config.projectPattern}.`, + ); + return false; + } + + if ( + this.config.group && + !project.path_with_namespace!.startsWith(`${this.config.group}/`) + ) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it does not match the group pattern ${this.config.group}.`, + ); + return false; + } + + if ( + this.config.skipForkedRepos && + project.hasOwnProperty('forked_from_project') + ) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it is a forked project.`, + ); + return false; + } + + const customFallbackBranch = + this.config.fallbackBranch !== 'master' + ? this.config.fallbackBranch + : undefined; + + const project_branch = + this.config.branch ?? + customFallbackBranch ?? + project.default_branch ?? + this.config.fallbackBranch; + + const hasFile = await client.hasFile( + project.path_with_namespace ?? '', + project_branch, + this.config.catalogFile, + ); + + return hasFile; + } } diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 8dffdf1d7b..b5a049538d 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -23,10 +23,17 @@ import { import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; -import { graphql, rest } from 'msw'; +import { DefaultEventsService } from '@backstage/plugin-events-node'; import { setupServer } from 'msw/node'; +import { handlers } from '../__testUtils__/handlers'; +import * as mock from '../__testUtils__/mocks'; +import { GroupNameTransformerOptions } from '../lib/types'; import { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider'; +const server = setupServer(...handlers); +setupRequestMockHandlers(server); +afterEach(() => jest.resetAllMocks()); + class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -42,13 +49,8 @@ class PersistingTaskRunner implements TaskRunner { const logger = getVoidLogger(); -const server = setupServer(); - -describe('GitlabOrgDiscoveryEntityProvider', () => { - setupRequestMockHandlers(server); - afterEach(() => jest.resetAllMocks()); - - it('no provider config', () => { +describe('GitlabOrgDiscoveryEntityProvider - configuration', () => { + it('should not instantiate providers when no config found', () => { const schedule = new PersistingTaskRunner(); const config = new ConfigReader({}); const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { @@ -59,61 +61,89 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { expect(providers).toHaveLength(0); }); - it('single simple discovery config with org disabled', () => { + it('should throw error when no matching GitLab integration config found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); - const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - schedule, - }); + const config = new ConfigReader(mock.config_github_host); - expect(providers).toHaveLength(0); + expect(() => { + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow('No gitlab integration found that matches host example.com'); }); - it('single simple discovery config with org enabled', () => { + it('should throw error when org configuration not found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - }, - }, - }, + const config = new ConfigReader(mock.config_no_org_integration); + + expect(() => { + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow('Org not enabled for test-id'); + }); + + it('should throw error when saas without group configuration', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader(mock.config_saas_no_group); + + expect(() => { + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow( + `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`, + ); + }); + it('should fail without schedule and scheduler', () => { + const config = new ConfigReader(mock.config_org_integration_saas); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('should fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader(mock.config_org_integration_saas); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:test-id', + ); + }); + + it('should instantiate provider with single simple saas config', async () => { + const schedule = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + const config = new ConfigReader(mock.config_org_integration_saas_sched); + const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + }); + + it('should instantiate provider when org enabled', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader(mock.config_org_integration); const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -125,35 +155,9 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); }); - it('multiple discovery configs', () => { + it('should instantiate providers when multiple valid provider configs', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - 'second-test': { - host: 'test-gitlab', - group: 'second-group', - orgEnabled: true, - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_org_double_integration); const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -167,29 +171,11 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { 'GitlabOrgDiscoveryEntityProvider:second-test', ); }); +}); - it('apply full update on scheduled execution', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - orgEnabled: true, - }, - }, - }, - }, - }); +describe('GitlabOrgDiscoveryEntityProvider - refresh', () => { + it('should apply full update on scheduled execution', async () => { + const config = new ConfigReader(mock.config_org_integration); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -203,185 +189,6 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { 'GitlabOrgDiscoveryEntityProvider:test-id', ); - server.use( - rest.get( - `https://api.gitlab.example/api/v4/groups/test-group/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.get(`https://api.gitlab.example/api/v4/users`, (_req, res, ctx) => { - const response = [ - { - id: 1, - username: 'test1', - name: 'Test Testit', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.example/test1', - created_at: '2023-01-19T07:27:03.333Z', - bio: '', - location: null, - public_email: null, - skype: '', - linkedin: '', - twitter: '', - website_url: '', - organization: null, - job_title: '', - pronouns: null, - bot: false, - work_information: null, - followers: 0, - following: 0, - is_followed: false, - local_time: null, - last_sign_in_at: '2023-01-19T07:27:49.601Z', - confirmed_at: '2023-01-19T07:27:02.905Z', - last_activity_on: '2023-01-19', - email: 'test@example.com', - theme_id: 1, - color_scheme_id: 1, - projects_limit: 100000, - current_sign_in_at: '2023-01-19T09:09:10.676Z', - identities: [], - can_create_group: true, - can_create_project: true, - two_factor_enabled: false, - external: false, - private_profile: false, - commit_email: 'test@example.com', - is_admin: false, - note: '', - }, - { - id: 2, - username: 'test2', - name: '', - state: 'active', - avatar_url: '', - web_url: 'https://gitlab.example/test2', - created_at: '2023-01-19T07:27:03.333Z', - bio: '', - location: null, - public_email: null, - skype: '', - linkedin: '', - twitter: '', - website_url: '', - organization: null, - job_title: '', - pronouns: null, - bot: false, - work_information: null, - followers: 0, - following: 0, - is_followed: false, - local_time: null, - last_sign_in_at: '2023-01-19T07:27:49.601Z', - confirmed_at: '2023-01-19T07:27:02.905Z', - last_activity_on: '2023-01-19', - email: 'test@example.com', - theme_id: 1, - color_scheme_id: 1, - projects_limit: 100000, - current_sign_in_at: '2023-01-19T09:09:10.676Z', - identities: [], - can_create_group: true, - can_create_project: true, - two_factor_enabled: false, - external: false, - private_profile: false, - commit_email: 'test@example.com', - is_admin: false, - note: '', - }, - ]; - return res(ctx.json(response)); - }), - rest.get(`https://api.gitlab.example/api/v4/groups`, (_req, res, ctx) => { - const response = [ - { - id: 1, - web_url: 'https://gitlab.example/groups/group1', - name: 'group1', - path: 'group1', - description: '', - visibility: 'internal', - share_with_group_lock: false, - require_two_factor_authentication: false, - two_factor_grace_period: 48, - project_creation_level: 'developer', - auto_devops_enabled: null, - subgroup_creation_level: 'owner', - emails_disabled: null, - mentions_disabled: null, - lfs_enabled: true, - default_branch_protection: 2, - avatar_url: null, - request_access_enabled: false, - full_name: '8020', - full_path: '8020', - created_at: '2017-06-19T06:42:34.160Z', - parent_id: null, - }, - { - id: 2, - web_url: 'https://gitlab.example/groups/group1/group2', - name: 'group2', - path: 'group1/group2', - description: 'Group2', - visibility: 'internal', - share_with_group_lock: false, - require_two_factor_authentication: false, - two_factor_grace_period: 48, - project_creation_level: 'developer', - auto_devops_enabled: null, - subgroup_creation_level: 'owner', - emails_disabled: null, - mentions_disabled: null, - lfs_enabled: true, - request_access_enabled: false, - full_name: 'group2', - full_path: 'group1/group2', - created_at: '2017-12-07T13:20:40.675Z', - parent_id: null, - }, - ]; - return res(ctx.json(response)); - }), - graphql - .link('https://test-gitlab/api/graphql') - .operation(async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: - req.variables.group === 'group1/group2' - ? [{ user: { id: 'gid://gitlab/User/1' } }] - : [], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); - await provider.connect(entityProviderConnection); const taskDef = schedule.getTasks()[0]; @@ -390,115 +197,15 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); await (taskDef.fn as () => Promise)(); - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://test-gitlab/test1', - 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/test1', - 'test-gitlab/user-login': 'https://gitlab.example/test1', - }, - name: 'test1', - }, - spec: { - memberOf: ['group1-group2'], - profile: { - displayName: 'Test Testit', - email: 'test@example.com', - picture: 'https://secure.gravatar.com/', - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://test-gitlab/test2', - 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/test2', - 'test-gitlab/user-login': 'https://gitlab.example/test2', - }, - name: 'test2', - }, - spec: { - memberOf: [], - profile: { - displayName: undefined, - email: 'test@example.com', - picture: undefined, - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://test-gitlab/group1/group2', - 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/group1/group2', - 'test-gitlab/team-path': 'group1/group2', - 'test-gitlab/visibility': 'internal', - }, - description: 'Group2', - name: 'group1-group2', - }, - spec: { - children: [], - profile: { - displayName: 'group2', - }, - type: 'team', - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - ]; - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: expectedEntities, + entities: mock.expected_full_org_scan_entities, }); }); - it('apply full update on scheduled execution for gitlab.com', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'gitlab.com', - apiBaseUrl: 'https://gitlab.com/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'gitlab.com', - group: 'group1', - orgEnabled: true, - }, - }, - }, - }, - }); + it('should exclude inactive user', async () => { + const config = new ConfigReader(mock.config_org_integration); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -512,126 +219,70 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { 'GitlabOrgDiscoveryEntityProvider:test-id', ); - server.use( - graphql - .link('https://gitlab.com/api/graphql') - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.data({ - group: { - descendantGroups: { - nodes: [ - { - id: 'gid://gitlab/Group/456', - name: 'group2', - description: 'Group2', - fullPath: 'group1/group2', - visibility: 'private', - parent: { - id: 'gid://gitlab/Group/123', - }, - }, - { - id: 'gid://gitlab/Group/789', - name: 'group3', - description: 'Group3', - fullPath: 'group1/group3', - visibility: 'public', - parent: { - id: 'gid://gitlab/Group/123', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - graphql - .link('https://gitlab.com/api/graphql') - .query('getGroupMembers', async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: - req.variables.group === 'group1/group2' - ? [{ user: { id: 'gid://gitlab/User/12' } }] - : [{ user: { id: 'gid://gitlab/User/34' } }], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - rest.get( - `https://gitlab.com/api/v4/groups/group1/members/all`, - (_req, res, ctx) => { - const response = [ - { - access_level: 30, - created_at: '2023-07-17T08:58:34.984Z', - expires_at: null, - id: 12, - username: 'testuser1', - name: 'Test User 1', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.com/testuser1', - email: 'testuser1@example.com', - group_saml_identity: { - provider: 'group_saml', - extern_uid: '51', - saml_provider_id: 1, - }, - is_using_seat: true, - membership_state: 'active', - }, - { - access_level: 30, - created_at: '2023-07-19T08:58:34.984Z', - expires_at: null, - id: 34, - username: 'testuser2', - name: 'Test User 2', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.com/testuser2', - email: 'testuser2@example.com', - group_saml_identity: { - provider: 'group_saml', - extern_uid: '52', - saml_provider_id: 1, - }, - is_using_seat: true, - membership_state: 'active', - }, - { - access_level: 50, - created_at: '2023-07-15T08:58:34.984Z', - expires_at: '2023-10-26', - id: 54, - username: 'group_100_bot_23dc8057bef66e05181f39be4652577c', - name: 'Token Bot', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: - 'https://gitlab.com/group_100_bot_23dc8057bef66e05181f39be4652577c', - group_saml_identity: null, - is_using_seat: false, - membership_state: 'active', - }, - ]; - return res(ctx.json(response)); - }, - ), + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', + ); + await (taskDef.fn as () => Promise)(); + const userEntities = mock.expected_full_org_scan_entities.filter( + entity => entity.entity.kind === 'User', + ); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(userEntities).not.toHaveLength(mock.all_users_response.length); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_full_org_scan_entities, + }); + }); + + it('should exclude user when email or username does not match userPattern', async () => { + const config = new ConfigReader(mock.config_userPattern_integration); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + + await (taskDef.fn as () => Promise)(); + const userEntities = mock.expected_full_org_scan_entities.filter( + element => element.entity.metadata.name !== 'MarioMario', + ); // filter out user with non matched e-mail + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(userEntities).not.toHaveLength(mock.all_users_response.length); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: userEntities, + }); + }); + + it('should apply full update on scheduled execution for gitlab.com', async () => { + const config = new ConfigReader(mock.config_org_integration_saas); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', ); await provider.connect(entityProviderConnection); @@ -642,263 +293,340 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); await (taskDef.fn as () => Promise)(); - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/testuser1', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/testuser1', - 'gitlab.com/user-login': 'https://gitlab.com/testuser1', - 'gitlab.com/saml-external-uid': '51', - }, - name: 'testuser1', - }, - spec: { - memberOf: ['group2'], - profile: { - displayName: 'Test User 1', - email: 'testuser1@example.com', - picture: 'https://secure.gravatar.com/', - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/testuser2', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/testuser2', - 'gitlab.com/user-login': 'https://gitlab.com/testuser2', - 'gitlab.com/saml-external-uid': '52', - }, - name: 'testuser2', - }, - spec: { - memberOf: ['group3'], - profile: { - displayName: 'Test User 2', - email: 'testuser2@example.com', - picture: 'https://secure.gravatar.com/', - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/group1/group2', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/group1/group2', - 'gitlab.com/team-path': 'group1/group2', - 'gitlab.com/visibility': 'private', - }, - description: 'Group2', - name: 'group2', - }, - spec: { - children: [], - profile: { - displayName: 'group2', - }, - type: 'team', - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/group1/group3', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/group1/group3', - 'gitlab.com/team-path': 'group1/group3', - 'gitlab.com/visibility': 'public', - }, - description: 'Group3', - name: 'group3', - }, - spec: { - children: [], - profile: { - displayName: 'group3', - }, - type: 'team', - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - ]; - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: expectedEntities, + entities: mock.expected_full_org_scan_entities_saas, + }); + }); +}); + +describe('GitlabOrgDiscoveryEntityProvider with events support', () => { + it('should ignore gitlab.group_destroy event when group pattern does not match', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_destroy_event_unmatched); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should apply a delta mutation if gitlab.group_destroy event received and defaultGroupTransformation in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_destroy_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: mock.expected_group_entity, + }); + }); + + it('should ignore gitlab.group_create event when group pattern does not match', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + await events.publish(mock.group_create_event_unmatched); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + it('should apply a delta mutation if gitlab.group_create event received and defaultGroupTransformation in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + await events.publish(mock.group_create_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_group_entity, + removed: [], + }); + }); + + it('should apply a delta mutation if gitlab.group_create event received and user/group transformations in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + function customGroupNameTransformer( + options: GroupNameTransformerOptions, + ): string { + return `${options.group.id}`; + } + + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + groupNameTransformer: customGroupNameTransformer, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_create_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_transformed_group_entity, + removed: [], + }); + }); + + it('should apply a delta mutation if gitlab.group_rename event received and defaultGroupTransformation in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_rename_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_added_group_entity, + removed: mock.expected_removed_group_entity, + }); + }); + + it('should apply a delta mutation if gitlab.user_create event received', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_create_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_single_user_entity, + removed: [], + }); + }); + + it('should apply a delta mutation if gitlab.user_destroy event received', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_destroy_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: mock.expected_single_user_removed_entity, + }); + }); + + it('should apply a delta mutation if gitlab.user_add_to_group event received and defaultGroupTransformer', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_add_to_group_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_group_user_entity, + removed: [], + }); + }); + + it('should ignore a delta mutation if gitlab.user_add_to_group event received and group outside scope', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_add_to_group_event_mismatched); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should apply a delta mutation if gitlab.user_remove_from_group event received and customGroupTransformer', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + function customGroupNameTransformer( + options: GroupNameTransformerOptions, + ): string { + return `${options.group.id}`; + } + + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + groupNameTransformer: customGroupNameTransformer, + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_remove_from_group_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_removed_user_entity, + removed: [], }); }); - - it('fail without schedule and scheduler', () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - }, - }, - }, - }); - - expect(() => - GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - }), - ).toThrow('Either schedule or scheduler must be provided'); - }); - - it('fail with scheduler but no schedule config', () => { - const scheduler = { - createScheduledTaskRunner: (_: any) => jest.fn(), - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - }, - }, - }, - }); - - expect(() => - GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ).toThrow( - 'No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:test-id', - ); - }); - - it('fail with scheduler but no group config when host is gitlab.com', () => { - const scheduler = { - createScheduledTaskRunner: (_: any) => jest.fn(), - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'gitlab.com', - apiBaseUrl: 'https://gitlab.com/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'gitlab.com', - orgEnabled: true, - }, - }, - }, - }, - }); - - expect(() => - GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ).toThrow( - `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`, - ); - }); - - it('single simple provider config with schedule in config', async () => { - const schedule = new PersistingTaskRunner(); - const scheduler = { - createScheduledTaskRunner: (_: any) => schedule, - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - schedule: { - frequency: 'PT30M', - timeout: 'PT3M', - }, - }, - }, - }, - }, - }); - const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }); - - expect(providers).toHaveLength(1); - expect(providers[0].getProviderName()).toEqual( - 'GitlabOrgDiscoveryEntityProvider:test-id', - ); - }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 90a7a14b6b..0135c7b4a9 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -25,6 +25,7 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; +import { EventsService } from '@backstage/plugin-events-node'; import { merge } from 'lodash'; import * as uuid from 'uuid'; import { LoggerService } from '@backstage/backend-plugin-api'; @@ -35,21 +36,24 @@ import { paginated, readGitlabConfigs, } from '../lib'; -import { - GitLabGroup, - GitLabUser, - GroupNameTransformer, - GroupTransformer as GroupEntitiesTransformer, - PagedResponse, - UserTransformer, -} from '../lib/types'; import { defaultGroupEntitiesTransformer, defaultGroupNameTransformer, defaultUserTransformer, } from '../lib/defaultTransformers'; +import { + GitLabGroup, + GitLabUser, + GroupTransformer as GroupEntitiesTransformer, + GroupNameTransformer, + PagedResponse, + SystemHookBaseGroupEventsSchema, + SystemHookBaseMembershipEventsSchema, + SystemHookBaseUserEventsSchema, + UserTransformer, +} from '../lib/types'; -type Result = { +type UserResult = { scanned: number; matches: GitLabUser[]; }; @@ -59,6 +63,36 @@ type GroupResult = { matches: GitLabGroup[]; }; +type SystemHookGroupCreateOrDestroyEventSchema = + SystemHookBaseGroupEventsSchema & { + event_name: 'group_create' | 'group_destroy'; + }; + +type SystemHookGroupRenameEventSchema = SystemHookBaseGroupEventsSchema & { + event_name: 'group_rename'; + old_path: string; + old_full_path: string; +}; + +type SystemHookUserCreateOrDestroyEventSchema = + SystemHookBaseUserEventsSchema & { + event_name: 'user_create' | 'user_destroy'; + }; + +type SystemHookCreateOrDestroyMembershipEventsSchema = + SystemHookBaseMembershipEventsSchema & { + event_name: 'user_add_to_group' | 'user_remove_from_group'; + }; + +// System level events +const TOPIC_GROUP_CREATE = 'gitlab.group_create'; +const TOPIC_GROUP_DESTROY = 'gitlab.group_destroy'; +const TOPIC_GROUP_RENAME = 'gitlab.group_rename'; +const TOPIC_USER_CREATE = 'gitlab.user_create'; +const TOPIC_USER_DESTROY = 'gitlab.user_destroy'; +const TOPIC_USER_ADD_GROUP = 'gitlab.user_add_to_group'; +const TOPIC_USER_REMOVE_GROUP = 'gitlab.user_remove_from_group'; + /** * Discovers users and groups from a Gitlab instance. * @public @@ -67,16 +101,19 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private readonly config: GitlabProviderConfig; private readonly integration: GitLabIntegration; private readonly logger: LoggerService; + private readonly events?: EventsService; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; private userTransformer: UserTransformer; private groupEntitiesTransformer: GroupEntitiesTransformer; private groupNameTransformer: GroupNameTransformer; + private readonly gitLabClient: GitLabClient; static fromConfig( config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; userTransformer?: UserTransformer; @@ -96,7 +133,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const integration = integrations.byHost(providerConfig.host); if (!providerConfig.orgEnabled) { - return; + throw new Error(`Org not enabled for ${providerConfig.id}.`); } if (!integration) { @@ -137,6 +174,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { config: GitlabProviderConfig; integration: GitLabIntegration; logger: LoggerService; + events?: EventsService; taskRunner: TaskRunner; userTransformer?: UserTransformer; groupEntitiesTransformer?: GroupEntitiesTransformer; @@ -148,11 +186,18 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { target: this.getProviderName(), }); this.scheduleFn = this.createScheduleFn(options.taskRunner); + this.events = options.events; + this.userTransformer = options.userTransformer ?? defaultUserTransformer; this.groupEntitiesTransformer = options.groupEntitiesTransformer ?? defaultGroupEntitiesTransformer; this.groupNameTransformer = options.groupNameTransformer ?? defaultGroupNameTransformer; + + this.gitLabClient = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); } getProviderName(): string { @@ -162,6 +207,117 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { async connect(connection: EntityProviderConnection): Promise { this.connection = connection; await this.scheduleFn(); + + // Specifies which topics will be listened to. + // The topics from the original GitLab events contain only the string 'gitlab'. These are caught by the GitlabEventRouter Module, who republishes them with a more specific topic 'gitlab.' + if (this.events) { + await this.events.subscribe({ + id: this.getProviderName(), + topics: [ + TOPIC_GROUP_CREATE, + TOPIC_GROUP_DESTROY, + TOPIC_GROUP_RENAME, + TOPIC_USER_CREATE, + TOPIC_USER_DESTROY, + TOPIC_USER_ADD_GROUP, + TOPIC_USER_REMOVE_GROUP, + ], + onEvent: async params => { + this.logger.info(`Received event from topic ${params.topic}`); + + const addEntitiesOperation = (entities: Entity[]) => ({ + removed: [], + added: entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), + })), + }); + + const removeEntitiesOperation = (entities: Entity[]) => ({ + added: [], + removed: entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), + })), + }); + + const replaceEntitiesOperation = (entities: Entity[]) => { + const entitiesToReplace = entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), + })); + + return { + removed: entitiesToReplace, + added: entitiesToReplace, + }; + }; + + // handle group change events + if ( + params.topic === TOPIC_GROUP_CREATE || + params.topic === TOPIC_GROUP_DESTROY + ) { + const payload: SystemHookGroupCreateOrDestroyEventSchema = + params.eventPayload as SystemHookGroupCreateOrDestroyEventSchema; + + const createDeltaOperation = + params.topic === TOPIC_GROUP_CREATE + ? addEntitiesOperation + : removeEntitiesOperation; + + await this.onGroupChange(payload, createDeltaOperation); + } + if (params.topic === TOPIC_GROUP_RENAME) { + const payload: SystemHookGroupRenameEventSchema = + params.eventPayload as SystemHookGroupRenameEventSchema; + + await this.onGroupEdit(payload, replaceEntitiesOperation); + } + + // handle user change events + if ( + params.topic === TOPIC_USER_CREATE || + params.topic === TOPIC_USER_DESTROY + ) { + const payload: SystemHookUserCreateOrDestroyEventSchema = + params.eventPayload as SystemHookUserCreateOrDestroyEventSchema; + + const createDeltaOperation = + params.topic === TOPIC_USER_CREATE + ? addEntitiesOperation + : removeEntitiesOperation; + + await this.onUserChange(payload, createDeltaOperation); + } + + // handle user membership changes + if ( + params.topic === TOPIC_USER_ADD_GROUP || + params.topic === TOPIC_USER_REMOVE_GROUP + ) { + const payload: SystemHookCreateOrDestroyMembershipEventsSchema = + params.eventPayload as SystemHookCreateOrDestroyMembershipEventsSchema; + + const createDeltaOperation = addEntitiesOperation; + + await this.onMembershipChange(payload, createDeltaOperation); + } + }, + }); + } } private createScheduleFn(taskRunner: TaskRunner): () => Promise { @@ -196,30 +352,33 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ); } - const client = new GitLabClient({ - config: this.integration.config, - logger: logger, - }); - let groups; let users; - if (client.isSelfManaged()) { - groups = paginated(options => client.listGroups(options), { - page: 1, - per_page: 100, - }); + if (this.gitLabClient.isSelfManaged()) { + groups = paginated( + options => this.gitLabClient.listGroups(options), + { + page: 1, + per_page: 100, + all_available: true, + }, + ); - users = paginated(options => client.listUsers(options), { - page: 1, - per_page: 100, - active: true, - }); + users = paginated( + options => this.gitLabClient.listUsers(options), + { + page: 1, + per_page: 100, + active: true, + }, + ); } else { - groups = (await client.listDescendantGroups(this.config.group)).items; + groups = (await this.gitLabClient.listDescendantGroups(this.config.group)) + .items; const rootGroup = this.config.group.split('/')[0]; users = paginated( - options => client.listSaaSUsers(rootGroup, options), + options => this.gitLabClient.listSaaSUsers(rootGroup, options), { page: 1, per_page: 100, @@ -229,7 +388,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const idMappedUser: { [userId: number]: GitLabUser } = {}; - const res: Result = { + const userRes: UserResult = { scanned: 0, matches: [], }; @@ -240,46 +399,45 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }; for await (const user of users) { - if (!this.config.userPattern.test(user.email ?? user.username ?? '')) { - continue; - } + userRes.scanned++; - res.scanned++; - - if (user.state !== 'active') { + if (!this.shouldProcessUser(user)) { + logger.debug(`Skipped user: ${user.username}`); continue; } idMappedUser[user.id] = user; - res.matches.push(user); + userRes.matches.push(user); } for await (const group of groups) { - if (!this.config.groupPattern.test(group.full_path ?? '')) { - continue; - } - - if ( - this.config.group && - !group.full_path.startsWith(`${this.config.group}/`) - ) { - continue; - } - groupRes.scanned++; + + if (!this.shouldProcessGroup(group)) { + logger.debug(`Skipped group: ${group.full_path}`); + continue; + } + logger.debug(`Processed group: ${group.full_path}`); + groupRes.matches.push(group); let groupUsers: PagedResponse = { items: [] }; try { - groupUsers = await client.getGroupMembers(group.full_path, ['DIRECT']); + const relations = this.config.allowInherited + ? ['DIRECT', 'INHERITED'] + : ['DIRECT']; + groupUsers = await this.gitLabClient.getGroupMembers( + group.full_path, + relations, + ); } catch (e) { logger.error( `Failed fetching users for group '${group.full_path}': ${e}`, ); } - for (const groupUser of groupUsers.items) { const user = idMappedUser[groupUser.id]; + if (user) { user.groups = (user.groups ?? []).concat(group); } @@ -288,13 +446,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const groupsWithUsers = groupRes.matches.filter(group => { return ( - res.matches.filter(x => { + userRes.matches.filter(x => { return !!x.groups?.find(y => y.id === group.id); }).length > 0 ); }); - const userEntities = res.matches.map(p => + const userEntities = userRes.matches.map(p => this.userTransformer({ user: p, integrationConfig: this.integration.config, @@ -309,6 +467,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupNameTransformer: this.groupNameTransformer, }); + logger.info( + `Scanned ${userRes.scanned} users and processed ${userEntities.length} users`, + ); + logger.info( + `Scanned ${groupRes.scanned} groups and processed ${groupEntities.length} groups`, + ); + await this.connection.applyMutation({ type: 'full', entities: [...userEntities, ...groupEntities].map(entity => ({ @@ -321,6 +486,282 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { })), }); } + private async onGroupChange( + event: SystemHookGroupCreateOrDestroyEventSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + let group: GitLabGroup | undefined; + if (event.event_name === 'group_destroy') { + group = { + id: event.group_id, + full_path: event.full_path, + name: event.name, + description: '', + parent_id: 0, + }; + } else { + group = await this.gitLabClient.getGroupById(event.group_id); + } + + if (!this.shouldProcessGroup(group)) { + this.logger.debug(`Skipped group ${group.full_path}.`); + return; + } + + // create the group entity + const groupEntity = this.groupEntitiesTransformer({ + groups: [group], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + // we need to fetch the parent group's object because its representation might be changed by the groupTransformer + if (group.parent_id) { + const parentGroup = await this.gitLabClient.getGroupById(group.parent_id); + + groupEntity[0].spec.parent = this.groupNameTransformer({ + group: parentGroup, + providerConfig: this.config, + }); + } + + this.logger.debug(`Applying mutation for group ${group.full_path}.`); + await this.connection.applyMutation({ + type: 'delta', + ...createDeltaOperation(groupEntity), + }); + } + + // the goal here is to trigger a mutation to remove the old entity and add the new one. + private async onGroupEdit( + event: SystemHookGroupRenameEventSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const groupToRemove: GitLabGroup = { + id: event.group_id, + full_path: event.old_full_path, + name: event.name, + description: '', + parent_id: 0, + }; + + if (!this.shouldProcessGroup(groupToRemove)) { + this.logger.debug(`Skipped group ${groupToRemove.full_path}.`); + return; + } + + const groupEntityToRemove = await this.groupEntitiesTransformer({ + groups: [groupToRemove], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + const groupToAdd = await this.gitLabClient.getGroupById(event.group_id); + + if (!this.shouldProcessGroup(groupToAdd)) { + this.logger.debug(`Skipped group ${groupToAdd.full_path}.`); + return; + } + + const groupEntityToAdd = await this.groupEntitiesTransformer({ + groups: [groupToAdd], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + if (groupToAdd.parent_id) { + const parentGroup = await this.gitLabClient.getGroupById( + groupToAdd.parent_id, + ); + + groupEntityToAdd[0].spec.parent = this.groupNameTransformer({ + group: parentGroup, + providerConfig: this.config, + }); + } + + const { added } = createDeltaOperation(groupEntityToAdd); + const { removed } = createDeltaOperation(groupEntityToRemove); + + this.logger.debug(`Applying mutation for group ${groupToAdd.full_path}.`); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onUserChange( + event: SystemHookUserCreateOrDestroyEventSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + let user: GitLabUser | undefined = undefined; + + // if user destroy event is received, retrieve user data from the event itself + if (event.event_name === 'user_destroy') { + user = { + id: event.user_id, + username: event.username, + email: event.email, + name: event.name, + state: 'active', // in the delete case it doesn't really matter if the user is active or not + web_url: '', + avatar_url: '', + }; + } + + // if user create event received fetch data from gitlab + if (event.event_name === 'user_create') { + user = await this.gitLabClient.getUserById(event.user_id); + } + + if (!user) { + this.logger.debug( + `Couldn't retrieve user data. Skipped ${event.event_name} event processing for user ${event.username}`, + ); + return; + } + + if (!this.shouldProcessUser(user)) { + this.logger.debug(`Skipped user ${user.username}.`); + return; + } + + const userEntity = await this.userTransformer({ + user: user, + integrationConfig: this.integration.config, + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + const { added, removed } = createDeltaOperation([userEntity]); + + this.logger.debug(`Applying mutation for user ${user.username}.`); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + // the goal here is to reconstruct the group either from which the user was removed or to which the user was added. Specifically, we add/remove the new user to/from the spec.member property array of the group entity. The Processor should take care of updating the relations + private async onMembershipChange( + event: SystemHookCreateOrDestroyMembershipEventsSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + // fetch group data from GitLab + const groupToRebuild: GitLabGroup = await this.gitLabClient.getGroupById( + event.group_id, + ); + + if (!groupToRebuild) { + this.logger.debug( + `Couldn't retrieve group data. Skipped ${event.event_name} event processing.`, + ); + return; + } + + // if the group is outside the scope there is no point creating anything related to it. + if (!this.shouldProcessGroup(groupToRebuild)) { + this.logger.debug(`Skipped group ${groupToRebuild.full_path}.`); + return; + } + + const relations = this.config.allowInherited + ? ['DIRECT', 'INHERITED'] + : ['DIRECT']; + + // fetch group members from GitLab + const groupMembers = await this.gitLabClient.getGroupMembers( + groupToRebuild.full_path, + relations, + ); + const usersToBeAdded = groupMembers.items; + + const groupEntityToModify = await this.groupEntitiesTransformer({ + groups: [groupToRebuild], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + // we need to fetch the parent group's object because its representation might be changed by the groupTransformer + if (groupToRebuild.parent_id) { + const parentGroup = await this.gitLabClient.getGroupById( + groupToRebuild.parent_id, + ); + + // update parent of the group entity + groupEntityToModify[0].spec.parent = this.groupNameTransformer({ + group: parentGroup, + providerConfig: this.config, + }); + } + + // update members of the group entity + groupEntityToModify[0].spec.members = + usersToBeAdded.length !== 0 ? usersToBeAdded.map(e => e.username) : []; + + const { added, removed } = createDeltaOperation(groupEntityToModify); + + this.logger.debug( + `Applying mutation for group ${groupToRebuild.full_path}.`, + ); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private shouldProcessGroup(group: GitLabGroup): boolean { + return ( + this.config.groupPattern.test(group.full_path) && + (!this.config.group || + group.full_path.startsWith(`${this.config.group}/`)) + ); + } + + private shouldProcessUser(user: GitLabUser): boolean { + return ( + this.config.userPattern.test(user.email ?? user.username ?? '') && + user.state === 'active' + ); + } private withLocations(host: string, baseUrl: string, entity: Entity): Entity { const location = diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index bac56cc6f3..09b42e6475 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -58,6 +58,7 @@ describe('config', () => { groupPattern: /[\s\S]*/, userPattern: /[\s\S]*/, orgEnabled: false, + allowInherited: false, schedule: undefined, skipForkedRepos: false, }), @@ -95,6 +96,7 @@ describe('config', () => { groupPattern: /[\s\S]*/, userPattern: /[\s\S]*/, orgEnabled: false, + allowInherited: false, schedule: undefined, skipForkedRepos: false, }), @@ -133,6 +135,7 @@ describe('config', () => { groupPattern: /[\s\S]*/, userPattern: /[\s\S]*/, orgEnabled: false, + allowInherited: false, schedule: undefined, skipForkedRepos: true, }), @@ -173,6 +176,7 @@ describe('config', () => { groupPattern: /[\s\S]*/, userPattern: /[\s\S]*/, orgEnabled: false, + allowInherited: false, skipForkedRepos: false, schedule: { frequency: Duration.fromISO('PT30M'), diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index c34cb81808..8606a525bd 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -43,6 +43,8 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { config.getOptionalString('groupPattern') ?? /[\s\S]*/, ); const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false; + const allowInherited: boolean = + config.getOptionalBoolean('allowInherited') ?? false; const skipForkedRepos: boolean = config.getOptionalBoolean('skipForkedRepos') ?? false; @@ -62,6 +64,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { groupPattern, schedule, orgEnabled, + allowInherited, skipForkedRepos, }; } diff --git a/yarn.lock b/yarn.lock index 70d2d0e684..af96b924bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5013,7 +5013,24 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab": +"@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend-test-utils": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + luxon: ^3.0.0 + languageName: unknown + linkType: soft + +"@backstage/plugin-catalog-backend-module-gitlab@workspace:^, @backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab" dependencies: @@ -5025,7 +5042,11 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend-test-utils": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + "@gitbeaker/rest": ^40.0.3 "@types/lodash": ^4.14.151 "@types/uuid": ^9.0.0 lodash: ^4.17.21 @@ -8602,6 +8623,17 @@ __metadata: languageName: node linkType: hard +"@gitbeaker/core@npm:^40.0.3": + version: 40.0.3 + resolution: "@gitbeaker/core@npm:40.0.3" + dependencies: + "@gitbeaker/requester-utils": ^40.0.3 + qs: ^6.11.2 + xcase: ^2.0.1 + checksum: 999cea3f1523eabfae473e8a21849fdd4f6b4d30f0de681e735b238713b198ff9381c35d2ba5949d27ddc7ed5a953b14fc5a6ed38a5d98b05b6a5670a0c69162 + languageName: node + linkType: hard + "@gitbeaker/node@npm:^35.1.0, @gitbeaker/node@npm:^35.8.0": version: 35.8.1 resolution: "@gitbeaker/node@npm:35.8.1" @@ -8638,6 +8670,18 @@ __metadata: languageName: node linkType: hard +"@gitbeaker/requester-utils@npm:^40.0.3": + version: 40.0.3 + resolution: "@gitbeaker/requester-utils@npm:40.0.3" + dependencies: + picomatch-browser: ^2.2.6 + qs: ^6.11.2 + rate-limiter-flexible: ^4.0.0 + xcase: ^2.0.1 + checksum: 1a46dc9c730c29e2229b04ed9bbe6cd87b7e5f16c1d0a5e78106f0287ab3d981ce2b099eb41cdb0f9180bb6b11d25f5f1fa92a91b69682bcd4ac986e58b55d31 + languageName: node + linkType: hard + "@gitbeaker/rest@npm:^39.25.0": version: 39.34.3 resolution: "@gitbeaker/rest@npm:39.34.3" @@ -8648,6 +8692,16 @@ __metadata: languageName: node linkType: hard +"@gitbeaker/rest@npm:^40.0.3": + version: 40.0.3 + resolution: "@gitbeaker/rest@npm:40.0.3" + dependencies: + "@gitbeaker/core": ^40.0.3 + "@gitbeaker/requester-utils": ^40.0.3 + checksum: 6a6427b82ee3357fbe07e1235de7a3c98d50155898180fbe4dd641c8575faa93853f1a39185e18876f84469e1a385ccd9beb26fd95b61c260b81eb0ed342355b + languageName: node + linkType: hard + "@google-cloud/container@npm:^5.0.0": version: 5.12.0 resolution: "@google-cloud/container@npm:5.12.0"