From a70377d3e1902480cbae14b2c7a861514f0a61ff Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Sat, 16 Mar 2024 12:15:00 +0100 Subject: [PATCH] feat: add events support for gitlab catalog entity provider Signed-off-by: ElaineDeMattosSilvaB --- .changeset/eleven-pandas-divide.md | 5 + .changeset/shy-students-clap.md | 5 + .../.eslintrc.js | 1 + .../README.md | 8 + .../api-report.md | 11 + .../catalog-info.yaml | 10 + .../package.json | 47 + ...leGitlabOrgDiscoveryEntityProvider.test.ts | 109 ++ ...gModuleGitlabOrgDiscoveryEntityProvider.ts | 55 + .../src/index.ts | 22 + .../CHANGELOG.md | 12 +- .../api-report.md | 5 +- .../package.json | 56 +- .../src/__testUtils__/handlers.ts | 521 +++++ .../src/__testUtils__/mocks.ts | 1721 +++++++++++++++++ .../src/lib/client.test.ts | 905 +++------ .../src/lib/client.ts | 77 +- .../src/lib/defaultTransformers.ts | 4 - .../src/lib/index.ts | 6 +- .../src/lib/types.ts | 79 +- ...oduleGitlabDiscoveryEntityProvider.test.ts | 28 +- ...alogModuleGitlabDiscoveryEntityProvider.ts | 12 +- .../GitlabDiscoveryEntityProvider.test.ts | 819 +++----- .../GitlabDiscoveryEntityProvider.ts | 331 +++- .../GitlabOrgDiscoveryEntityProvider.test.ts | 1254 +++++------- .../GitlabOrgDiscoveryEntityProvider.ts | 515 ++++- yarn.lock | 23 +- 27 files changed, 4597 insertions(+), 2044 deletions(-) create mode 100644 .changeset/eleven-pandas-divide.md create mode 100644 .changeset/shy-students-clap.md create mode 100644 plugins/catalog-backend-module-gitlab-org/.eslintrc.js create mode 100644 plugins/catalog-backend-module-gitlab-org/README.md create mode 100644 plugins/catalog-backend-module-gitlab-org/api-report.md create mode 100644 plugins/catalog-backend-module-gitlab-org/catalog-info.yaml create mode 100644 plugins/catalog-backend-module-gitlab-org/package.json create mode 100644 plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts create mode 100644 plugins/catalog-backend-module-gitlab-org/src/index.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts diff --git a/.changeset/eleven-pandas-divide.md b/.changeset/eleven-pandas-divide.md new file mode 100644 index 0000000000..912d34570a --- /dev/null +++ b/.changeset/eleven-pandas-divide.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Added events support for `GitlabDiscoveryEntityProvider` and `GitlabOrgDiscoveryEntityProvider`. diff --git a/.changeset/shy-students-clap.md b/.changeset/shy-students-clap.md new file mode 100644 index 0000000000..0e6244f21d --- /dev/null +++ b/.changeset/shy-students-clap.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab-org': patch +--- + +Added a new `catalog-backend-module-gitlab-org` module which adds the `GitlabOrgDiscoveryEntityProvider` to the catalog's providers using the new backend system. diff --git a/plugins/catalog-backend-module-gitlab-org/.eslintrc.js b/plugins/catalog-backend-module-gitlab-org/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-gitlab-org/README.md b/plugins/catalog-backend-module-gitlab-org/README.md new file mode 100644 index 0000000000..0b86b8d815 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/README.md @@ -0,0 +1,8 @@ +# Catalog Backend Module for GitLab Organizational Data + +This is an extension module to the plugin-catalog-backend plugin that provides group and user entities by scrapping or receiving events from a GitLab instance. + +## Getting started + +See [Backstage documentation](https://backstage.io/docs/integrations/gitlab/org) for details on how to install +and configure the plugin. diff --git a/plugins/catalog-backend-module-gitlab-org/api-report.md b/plugins/catalog-backend-module-gitlab-org/api-report.md new file mode 100644 index 0000000000..2a36f49206 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/api-report.md @@ -0,0 +1,11 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-gitlab-org" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; + +// @public +const catalogModuleGitlabOrgDiscoveryEntityProvider: () => BackendFeature; +export default catalogModuleGitlabOrgDiscoveryEntityProvider; +``` diff --git a/plugins/catalog-backend-module-gitlab-org/catalog-info.yaml b/plugins/catalog-backend-module-gitlab-org/catalog-info.yaml new file mode 100644 index 0000000000..1846e3a9df --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-catalog-backend-module-gitlab-org + title: '@backstage/plugin-catalog-backend-module-gitlab-org' + description: The gitlab-org backend module for the catalog plugin. +spec: + lifecycle: experimental + type: backstage-backend-plugin-module + owner: maintainers diff --git a/plugins/catalog-backend-module-gitlab-org/package.json b/plugins/catalog-backend-module-gitlab-org/package.json new file mode 100644 index 0000000000..7d92fda325 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/package.json @@ -0,0 +1,47 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-gitlab-org", + "version": "0.0.0", + "description": "The gitlab-org backend module for the catalog plugin.", + "backstage": { + "role": "backend-plugin-module" + }, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-gitlab-org" + }, + "license": "Apache-2.0", + "main": "src/index.ts", + "types": "src/index.ts", + "files": [ + "dist" + ], + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^" + }, + "devDependencies": { + "@backstage/backend-tasks": "workspace:^", + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@backstage/plugin-events-backend-test-utils": "workspace:^", + "luxon": "^3.0.0" + } +} diff --git a/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts new file mode 100644 index 0000000000..568c91c8e2 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createServiceFactory } from '@backstage/backend-plugin-api'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { TestEventsService } from '@backstage/plugin-events-backend-test-utils'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; +import { Duration } from 'luxon'; +import { catalogModuleGitlabOrgDiscoveryEntityProvider } from './catalogModuleGitlabOrgDiscoveryEntityProvider'; + +describe('catalogModuleGitlabOrgDiscoveryEntityProvider', () => { + it('should register provider at the catalog extension point', async () => { + const events = new TestEventsService(); + const eventsServiceFactory = createServiceFactory({ + service: eventsServiceRef, + deps: {}, + async factory({}) { + return events; + }, + }); + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const connection = jest.fn() as unknown as EntityProviderConnection; + const runner = jest.fn(); + const scheduler = mockServices.scheduler.mock({ + createScheduledTaskRunner(schedule) { + usedSchedule = schedule; + return { run: runner }; + }, + }); + + const config = { + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + orgEnabled: true, + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }; + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + features: [ + eventsServiceFactory(), + catalogModuleGitlabOrgDiscoveryEntityProvider(), + mockServices.rootConfig.factory({ data: config }), + mockServices.logger.factory(), + scheduler.factory, + ], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(runner).not.toHaveBeenCalled(); + + const provider = addedProviders!.pop()!; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + await provider.connect(connection); + expect(events.subscribed).toHaveLength(1); + expect(events.subscribed[0].id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + expect(runner).toHaveBeenCalledTimes(1); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts new file mode 100644 index 0000000000..ffb2cfb629 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; + +/** + * Registers the GitlabOrgDiscoveryEntityProvider with the catalog processing extension point. + * + * @public + */ +export const catalogModuleGitlabOrgDiscoveryEntityProvider = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'gitlabOrgDiscoveryEntityProvider', + register(env) { + env.registerInit({ + deps: { + config: coreServices.rootConfig, + catalog: catalogProcessingExtensionPoint, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + events: eventsServiceRef, + }, + async init({ config, catalog, logger, scheduler, events }) { + const gitlabOrgDiscoveryEntityProvider = + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger: loggerToWinstonLogger(logger), + events, + scheduler, + }); + catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider); + }, + }); + }, + }); diff --git a/plugins/catalog-backend-module-gitlab-org/src/index.ts b/plugins/catalog-backend-module-gitlab-org/src/index.ts new file mode 100644 index 0000000000..bbd5f1f4c3 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/src/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The gitlab-org backend module for the catalog plugin. + * + * @packageDocumentation + */ +export { catalogModuleGitlabOrgDiscoveryEntityProvider as default } from './catalogModuleGitlabOrgDiscoveryEntityProvider'; diff --git a/plugins/catalog-backend-module-gitlab/CHANGELOG.md b/plugins/catalog-backend-module-gitlab/CHANGELOG.md index 4e8493d5d6..92f0ea7f21 100644 --- a/plugins/catalog-backend-module-gitlab/CHANGELOG.md +++ b/plugins/catalog-backend-module-gitlab/CHANGELOG.md @@ -385,8 +385,8 @@ ref: - https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api - https://docs.gitlab.com/ee/api/members.html#limitations + + - 890e3b5ad4: Make sure to include the error message when ingestion fails - 0b55f773a7: Removed some unused dependencies @@ -417,8 +417,8 @@ ref: - https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api - https://docs.gitlab.com/ee/api/members.html#limitations + + - 0b55f773a7: Removed some unused dependencies - Updated dependencies @@ -1211,7 +1211,7 @@ - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. Please find how to configure the schedule at the config at - https://backstage.io/docs/integrations/gitlab/discovery + - 4c9f7847e4: Updated dependency `msw` to `^0.48.0` while moving it to be a dev dependency. - Updated dependencies @@ -1250,7 +1250,7 @@ - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. Please find how to configure the schedule at the config at - https://backstage.io/docs/integrations/gitlab/discovery + - Updated dependencies - @backstage/backend-common@0.16.0-next.0 diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index cd9ade4bfb..809ac168f2 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -8,6 +8,7 @@ import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { EventsService } from '@backstage/plugin-events-node'; import { GitLabIntegrationConfig } from '@backstage/integration'; import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/plugin-catalog-node'; @@ -26,6 +27,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; }, @@ -63,7 +65,6 @@ export type GitLabGroup = { name: string; full_path: string; description?: string; - visibility?: string; parent_id?: number; }; @@ -81,6 +82,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; userTransformer?: UserTransformer; @@ -103,6 +105,7 @@ export type GitlabProviderConfig = { projectPattern: RegExp; userPattern: RegExp; groupPattern: RegExp; + allowInherited?: boolean; orgEnabled?: boolean; schedule?: TaskScheduleDefinition; skipForkedRepos?: boolean; diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 75bccd4faa..54ecf67503 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -1,5 +1,21 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ { "name": "@backstage/plugin-catalog-backend-module-gitlab", + "version": "0.3.10-next.0", "description": "A Backstage catalog backend module that helps integrate towards GitLab", "version": "0.3.15", "main": "src/index.ts", @@ -8,11 +24,23 @@ "publishConfig": { "access": "public" }, + "keywords": [ + "backstage" + ], + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-gitlab" + }, + "license": "Apache-2.0", "exports": { ".": "./src/index.ts", "./alpha": "./src/alpha.ts", "./package.json": "./package.json" }, + "main": "src/index.ts", + "types": "src/index.ts", "typesVersions": { "*": { "alpha": [ @@ -23,26 +51,18 @@ ] } }, - "backstage": { - "role": "backend-plugin-module" - }, - "homepage": "https://backstage.io", - "repository": { - "type": "git", - "url": "https://github.com/backstage/backstage", - "directory": "plugins/catalog-backend-module-gitlab" - }, - "keywords": [ - "backstage" + "files": [ + "config.d.ts", + "dist" ], "scripts": { - "start": "backstage-cli package start", "build": "backstage-cli package build", + "clean": "backstage-cli package clean", "lint": "backstage-cli package lint", - "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", - "clean": "backstage-cli package clean" + "start": "backstage-cli package start", + "test": "backstage-cli package test" }, "dependencies": { "@backstage/backend-common": "workspace:^", @@ -51,7 +71,10 @@ "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/integration": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", + "@gitbeaker/rest": "^39.25.0", "lodash": "^4.17.21", "node-fetch": "^2.6.7", "uuid": "^9.0.0" @@ -59,14 +82,11 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", + "@backstage/plugin-events-backend-test-utils": "workspace:^", "@types/lodash": "^4.14.151", "@types/uuid": "^9.0.0", "luxon": "^3.0.0", "msw": "^1.0.0" }, - "files": [ - "config.d.ts", - "dist" - ], "configSchema": "config.d.ts" } diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts new file mode 100644 index 0000000000..2f464dc1fe --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts @@ -0,0 +1,521 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { graphql, rest } from 'msw'; +import { + all_groups_response, + all_projects_response, + all_saas_users_response, + all_users_response, + apiBaseUrl, + apiBaseUrlSaas, + graphQlBaseUrl, + paged_endpoint, + saasGraphQlBaseUrl, + some_endpoint, + unhealthy_endpoint, + userID, +} from './mocks'; + +const httpHandlers = [ + /** + * Project REST endpoint mocks + */ + + // fetch all projects in an instance + rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => { + return res(ctx.set('x-next-page', ''), ctx.json(all_projects_response)); + }), + + rest.get(`${apiBaseUrl}/projects/42`, (_, res, ctx) => { + return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); + }), + + // testing non existing file + rest.get( + `${apiBaseUrl}/projects/test-group%2Ftest-repo1/repository/files/catalog-info.yaml`, + (_, res, ctx) => { + return res(ctx.status(400), ctx.json({ error: 'Not found' })); + }, + ), + + /** + * Group REST endpoint mocks + */ + + rest.get(`${apiBaseUrl}/groups`, (_req, res, ctx) => { + return res(ctx.set('x-next-page', ''), ctx.json(all_groups_response)); + }), + + rest.get(`${apiBaseUrl}/groups/42`, (_, res, ctx) => { + return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); + }), + + rest.get(`${apiBaseUrlSaas}/groups/group1/members/all`, (_req, res, ctx) => { + return res(ctx.json(all_saas_users_response)); + }), + + /** + * Users REST endpoint mocks + */ + + rest.get(`${apiBaseUrl}/users`, (_req, res, ctx) => { + return res(ctx.set('x-next-page', ''), ctx.json(all_users_response)); + }), + + rest.get(`${apiBaseUrl}/users/${userID}`, (_, res, ctx) => { + return res(ctx.json(all_users_response.find(user => user.id === userID))); + }), + + rest.get(`${apiBaseUrl}/users/42`, (_, res, ctx) => { + return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); + }), + + /** + * others + */ + + // mock a 4 page response + rest.get(`${apiBaseUrl}${paged_endpoint}`, (req, res, ctx) => { + const page = req.url.searchParams.get('page'); + const currentPage = page ? Number(page) : 1; + const fakePageCount = 4; + + return res( + // set next page number header if page requested is less than count + ctx.set( + 'x-next-page', + currentPage < fakePageCount ? String(currentPage + 1) : '', + ), + ctx.json([{ someContentOfPage: currentPage }]), + ); + }), + + rest.get(`${apiBaseUrl}${unhealthy_endpoint}`, (_, res, ctx) => { + return res(ctx.status(400), ctx.json({ error: 'some error' })); + }), + + rest.get(`${apiBaseUrl}${some_endpoint}`, (req, res, ctx) => { + return res(ctx.json([{ endpoint: req.url.toString() }])); + }), +]; + +// dynamic handlers + +const httpGroupFindByIdDynamic = all_groups_response.map(group => { + return rest.get(`${apiBaseUrl}/groups/${group.id}`, (_, res, ctx) => { + return res(ctx.json(all_groups_response.find(g => g.id === group.id))); + }); +}); + +const httpGroupListDescendantProjectsById = all_groups_response.map(group => { + return rest.get( + `${apiBaseUrl}/groups/${group.id}/projects`, + (_, res, ctx) => { + const projectsInGroup = all_projects_response.filter(p => + p.path_with_namespace?.includes(group.name), + ); + + return res(ctx.json(projectsInGroup)); + }, + ); +}); + +const httpGroupListDescendantProjectsByName = all_groups_response.map(group => { + return rest.get( + `${apiBaseUrl}/groups/${group.name}/projects`, + (_, res, ctx) => { + const projectsInGroup = all_projects_response.filter(p => + p.path_with_namespace?.includes(group.name), + ); + + return res(ctx.json(projectsInGroup)); + }, + ); +}); +const httpGroupFindByNameDynamic = all_groups_response.map(group => { + return rest.get(`${apiBaseUrl}/groups/${group.name}`, (_, res, ctx) => { + return res(ctx.json(all_groups_response.find(g => g.name === group.name))); + }); +}); +const httpProjectFindByIdDynamic = all_projects_response.map(project => { + return rest.get(`${apiBaseUrl}/projects/${project.id}`, (_, res, ctx) => { + return res(ctx.json(all_projects_response.find(p => p.id === project.id))); + }); +}); +const httpProjectCatalogDynamic = all_projects_response.map(project => { + const path: string = project.path_with_namespace + ? project.path_with_namespace!.replace(/\//g, '%2F') + : `${project.path_with_namespace}%2F${project.name}`; + + return rest.head( + `${apiBaseUrl}/projects/${path}/repository/files/catalog-info.yaml`, + (req, res, ctx) => { + const branch = req.url.searchParams.get('ref'); + if (branch === project.default_branch) { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ); +}); + +/** + * GraphQL endpoint mocks + */ + +const graphqlHandlers = [ + graphql + .link(graphQlBaseUrl) + .query('getGroupMembers', async (req, res, ctx) => { + const { group, relations } = req.variables; + + if (group === 'group1' && relations.includes('DIRECT')) { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + if (group === 'saas-multi-user-group') { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + publicEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + + if (group === 'non-existing-group' || group === '') { + return res( + ctx.data({ + group: { + groupMembers: { + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + + if (group === 'multi-page' && relations.includes('DIRECT')) { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: req.variables.endCursor + ? [{ user: { id: 'gid://gitlab/User/2' } }] + : [{ user: { id: 'gid://gitlab/User/1' } }], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ); + } + + if (group === 'multi-page-saas') { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: req.variables.endCursor + ? [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ] + : [ + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + publicEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ); + } + + if (group === 'error-group') { + return res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ); + } + + return res(ctx.status(200)); + }), + + graphql + .link(graphQlBaseUrl) + .query('listDescendantGroups', async (req, res, ctx) => { + const { group } = req.variables; + + if (group === 'error-group') { + return res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ); + } + + if (group === 'group-with-parent') { + return res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/1', + name: 'group-with-parent', + description: 'description1', + fullPath: 'path/group-with-parent', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + if (group === 'root') { + return res( + ctx.data({ + group: { + descendantGroups: { + nodes: req.variables.endCursor + ? [ + { + id: 'gid://gitlab/Group/1', + name: 'group1', + description: 'description1', + fullPath: 'path/group1', + parent: { + id: '123', + }, + }, + ] + : [ + { + id: 'gid://gitlab/Group/2', + name: 'group2', + description: 'description2', + fullPath: 'path/group2', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ); + } + + if (group === 'non-existing-group') { + return res( + ctx.data({ + group: {}, + }), + ); + } + return res(ctx.status(200)); + }), + + graphql + .link(saasGraphQlBaseUrl) + .query('getGroupMembers', async (req, res, ctx) => { + const { group } = req.variables; + + if (group === 'saas-multi-user-group') { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + publicEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + + if (group === '') { + return res(ctx.data({})); + } + + if (group === 'error-group') { + return res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ); + } + + return res(ctx.status(200)); + }), + + graphql + .link(saasGraphQlBaseUrl) + .query('listDescendantGroups', async (_, res, ctx) => { + return res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/456', + name: 'group1', + description: 'Group1', + fullPath: 'group1/group1', + parent: { + id: 'gid://gitlab/Group/123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + }), +]; + +export const handlers = [ + ...httpHandlers, + ...httpProjectFindByIdDynamic, + ...httpProjectCatalogDynamic, + ...httpGroupFindByIdDynamic, + ...httpGroupFindByNameDynamic, + ...httpGroupListDescendantProjectsById, + ...httpGroupListDescendantProjectsByName, + ...graphqlHandlers, +]; diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts new file mode 100644 index 0000000000..9e0f55d05d --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts @@ -0,0 +1,1721 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { locationSpecToMetadataName } from '@backstage/plugin-catalog-node'; +import { EventParams } from '@backstage/plugin-events-node'; +import { GitLabGroup, GitLabProject, GitLabUser } from '../lib/types'; + +type MockObject = Record; + +/** + * General + */ +export const apiBaseUrl: string = 'https://example.com/api/v4'; +export const apiBaseUrlSaas: string = 'https://gitlab.com/api/v4'; +export const graphQlBaseUrl = 'https://example.com/api/graphql'; +export const saasGraphQlBaseUrl = 'https://gitlab.com/api/graphql'; +export const groupName: string = 'group1'; +export const groupID: number = 1; +export const userID: number = 1; +export const projectID: number = 1; + +/** + * Endpoints + */ + +export const paged_endpoint: string = `/paged-endpoint`; +export const some_endpoint: string = `/some-endpoint`; +export const unhealthy_endpoint = `/unhealthy-endpoint`; + +/** + * Integration Configurations + */ + +export const config_self_managed: MockObject = { + host: 'example.com', + token: 'test-token', + apiBaseUrl: 'https://example.com/api/v4', + baseUrl: 'https://example.com', +}; + +export const config_saas: MockObject = { + host: 'gitlab.com', + token: 'test-token', + apiBaseUrl: 'https://gitlab.com/api/v4', + baseUrl: 'https://gitlab.com', +}; + +export const config_no_org_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_disabled_org_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + skipForkedRepos: false, + orgEnabled: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_saas_no_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + skipForkedRepos: false, + orgEnabled: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_non_gitlab_host: MockObject = { + integrations: { + github: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_single_integration_branch: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + branch: 'main', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_single_integration_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'nonMatchingGroup', + branch: 'main', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_fallbackBranch_branch: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + fallbackBranch: 'staging', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_single_integration_skip_forks: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + skipForkedRepos: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_no_schedule: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + orgEnabled: true, + skipForkedRepos: true, + }, + }, + }, + }, +}; + +export const config_single_integration_project_pattern: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + projectPattern: 'test-repo', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_no_schedule_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + projectPattern: 'test-repo', + skipForkedRepos: true, + }, + }, + }, + }, +}; + +export const config_unmatched_project_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'my-other-group', + projectPattern: 'my-other-project', + skipForkedRepos: true, + }, + }, + }, + }, +}; + +export const config_double_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + }, + 'second-test': { + host: 'example.com', + group: 'second-group', + }, + }, + }, + }, +}; + +export const config_org_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_userPattern_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + userPattern: '^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+.[a-zA-Z]{2,}$', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_saas: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + group: 'group1', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_no_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_saas_no_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_saas_sched: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + group: 'group1', + groupPattern: '^group.*', + orgEnabled: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_org_double_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + orgEnabled: true, + }, + 'second-test': { + host: 'example.com', + group: 'second-group', + orgEnabled: true, + }, + }, + }, + }, +}; + +/** + * GitLab API responses + */ + +export const all_projects_response: GitLabProject[] = [ + { + id: 1, + description: 'Project One Description', + name: 'test-repo1', + default_branch: 'main', + path: 'test-repo1', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo1', + path_with_namespace: 'group1/test-repo1', + }, + { + id: 2, + description: 'Project Two Description', + name: 'test-repo2', + default_branch: 'prd', + path: 'test-repo2', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo2', + path_with_namespace: 'group1/test-repo2', + }, + // unmatched project + { + id: 3, + description: 'Project Three Description', + name: 'repo3', + default_branch: 'main', + path: 'repo3', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/repo3', + path_with_namespace: 'group1/repo3', + }, + // forked project + { + id: 4, + description: 'Project Four Description', + name: 'test-repo4-forked', + default_branch: 'main', + path: 'test-repo4-forked', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo4-forked', + path_with_namespace: 'group1/test-repo4-forked', + forked_from_project: { id: 13083 }, + }, + // fallBack branch + { + id: 5, + description: 'Project Five Description', + name: 'test-repo5-staging', + default_branch: 'staging', + path: 'test-repo5-staging', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo5-staging', + path_with_namespace: 'group1/test-repo5-staging', + }, + // diffrent group + { + id: 6, + description: 'Project Six Description', + name: 'test-repo6', + default_branch: 'main', + path: 'test-repo6', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo6', + path_with_namespace: 'awesome-group/test-repo6', + }, +]; + +export const all_users_response: GitLabUser[] = [ + { + id: 1, + username: 'JohnDoe', + name: 'John Doe', + state: 'active', + email: 'john.doe@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/john_doe', + }, + { + id: 2, + username: 'JaneDoe', + name: 'Jane Doe', + state: 'active', + email: 'jane.doe@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/jane_doe', + }, + { + id: 3, + username: 'MarySmith', + name: 'Mary Smith', + state: 'active', + email: 'mary.smith@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/mary_smith', + }, + // inactive user + { + id: 4, + username: 'LuigiMario', + name: 'Luigi Mario', + state: 'inactive', + email: 'luigi.mario@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/luigi_mario', + }, + // malfomed email address + { + id: 5, + username: 'MarioMario', + name: 'Mario Mario', + state: 'active', + email: 'mario.mario-company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/mario_mario', + }, +]; + +export const all_saas_users_response: MockObject[] = [ + { + access_level: 30, + created_at: '2023-07-17T08:58:34.984Z', + expires_at: null, + id: 12, + username: 'testuser1', + name: 'Test User 1', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.com/testuser1', + email: 'testuser1@example.com', + group_saml_identity: { + provider: 'group_saml', + extern_uid: '51', + saml_provider_id: 1, + }, + is_using_seat: true, + membership_state: 'active', + }, + { + access_level: 30, + created_at: '2023-07-19T08:58:34.984Z', + expires_at: null, + id: 34, + username: 'testuser2', + name: 'Test User 2', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.com/testuser2', + email: 'testuser2@example.com', + group_saml_identity: { + provider: 'group_saml', + extern_uid: '52', + saml_provider_id: 1, + }, + is_using_seat: true, + membership_state: 'active', + }, + { + access_level: 50, + created_at: '2023-07-15T08:58:34.984Z', + expires_at: '2023-10-26', + id: 54, + username: 'group_100_bot_23dc8057bef66e05181f39be4652577c', + name: 'Token Bot', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: + 'https://gitlab.com/group_100_bot_23dc8057bef66e05181f39be4652577c', + group_saml_identity: null, + is_using_seat: false, + membership_state: 'active', + }, +]; + +export const all_groups_response: GitLabGroup[] = [ + { + id: 1, + name: 'group1', + description: '', + full_path: 'group1', + }, + { + id: 2, + name: 'group2', + description: '', + full_path: 'group2', + }, + { + id: 3, + name: 'group3', + description: '', + full_path: 'group3', + }, + { + id: 4, + name: 'group1', + description: '', + full_path: 'group-new/group1', + }, + { + id: 5, + name: 'nonMatchingGroup', + description: '', + full_path: 'parent1/nonMatchingGroup', + }, +]; + +export const group_with_parent: MockObject[] = [ + { + id: 1, + name: 'group-with-parent', + description: 'description1', + full_path: 'path/group-with-parent', + parent_id: 123, + }, +]; + +export const expectedSaasMember: MockObject[] = [ + { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }, + { + id: 2, + username: 'user2', + email: 'user2@example.com', + name: 'user2', + state: 'active', + web_url: 'user2.com', + avatar_url: 'user2', + }, +]; + +export const expectedSaasGroup: MockObject[] = [ + { + id: 1, + name: 'group1', + description: 'description1', + full_path: 'path/group1', + parent_id: 123, + }, + { + id: 2, + name: 'group2', + description: 'description2', + full_path: 'path/group2', + parent_id: 123, + }, +]; + +/** + * GitLab Events + */ + +const added_commits: MockObject[] = [ + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: ['catalog-info.yaml'], + modified: [], + removed: [], + }, + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: ['cool-folder-1/cool-folder-2/catalog-info.yaml'], + modified: [], + removed: [], + }, +]; + +const removed_commits: MockObject[] = [ + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: [], + modified: [], + removed: ['catalog-info.yaml'], + }, +]; + +const modified_commits: MockObject[] = [ + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: [], + modified: ['catalog-info.yaml'], + removed: [], + }, +]; + +export const group_destroy_event: EventParams = { + topic: 'gitlab.group_destroy', + eventPayload: { + event_name: 'group_destroy', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'group1', + path: 'group1', + full_path: 'group1', + group_id: 123, + }, +}; + +export const group_destroy_event_unmatched: EventParams = { + topic: 'gitlab.group_destroy', + eventPayload: { + event_name: 'group_destroy', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'my-awesome-group', + path: 'my-awesome-group', + full_path: 'parent/my-awesome-group', + group_id: 123, + }, +}; + +export const group_rename_event: EventParams = { + topic: 'gitlab.group_rename', + eventPayload: { + event_name: 'group_rename', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'group1', // this is the displayname + path: 'group-new', + full_path: 'group-new/group1', + old_path: 'group-old', + old_full_path: 'group-old/group1', + group_id: 4, + }, +}; + +export const group_create_event: EventParams = { + topic: 'gitlab.group_create', + eventPayload: { + event_name: 'group_create', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'group1', + path: 'group1', + full_path: 'group1', + group_id: 1, + }, +}; +export const group_create_event_unmatched: EventParams = { + topic: 'gitlab.group_create', + eventPayload: { + event_name: 'group_create', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'nonMatchingGroup', + path: 'nonMatchingGroup', + full_path: 'parent1/nonMatchingGroup', + group_id: 5, + }, +}; + +export const user_create_event: EventParams = { + topic: 'gitlab.user_create', + eventPayload: { + event_name: 'user_create', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + email: 'john.doe@company.com', + name: 'John Doe', + username: 'johndoe', + user_id: 1, + }, +}; +export const user_destroy_event: EventParams = { + topic: 'gitlab.user_destroy', + eventPayload: { + event_name: 'user_destroy', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + email: 'john.doe@company.com', + name: 'John Doe', + username: 'johndoe', + user_id: 1, + }, +}; + +export const user_add_to_group_event: EventParams = { + topic: 'gitlab.user_add_to_group', + eventPayload: { + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + group_name: 'group1', + group_path: 'my-groups/group1', + group_id: 1, + user_username: 'JohnDoe', + user_name: 'John Doe', + user_email: 'john.doe@company.com', + user_id: 1, + group_access: 'Owner', + expires_at: null, + group_plan: null, + event_name: 'user_add_to_group', + }, +}; + +export const user_add_to_group_event_mismatched: EventParams = { + topic: 'gitlab.user_add_to_group', + eventPayload: { + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + group_name: 'nonMatchingGroup', + group_path: 'parent1/nonMatchingGroup', + group_id: 5, + user_username: 'JohnDoe', + user_name: 'John Doe', + user_email: 'john.doe@company.com', + user_id: 1, + group_access: 'Owner', + expires_at: null, + group_plan: null, + event_name: 'user_add_to_group', + }, +}; + +export const user_remove_from_group_event: EventParams = { + topic: 'gitlab.user_remove_from_group', + eventPayload: { + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + group_name: 'group1', + group_path: 'my-groups/group1', + group_id: 1, + user_username: 'user1', + user_name: 'John Doe', + user_email: 'john.doe@company.com', + user_id: 1, + group_access: 'Owner', + expires_at: null, + group_plan: null, + event_name: 'user_remove_from_group', + }, +}; + +export const push_add_event: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 1, + project: { + name: 'test-repo', + path_with_namespace: 'group1/test-repo1', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo1', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo1', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: added_commits, + total_commits_count: 2, + repository: { + name: 'test-repo1', + url: 'https://gitlab.com/group1/test-repo1', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo1', + git_http_url: 'https://gitlab.com/group1/test-repo1.git', + git_ssh_url: 'git@gitlab.com:group1/test-repo1.git', + visibility_level: 20, + }, + }, +}; + +export const push_add_event_unmatched_group: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 6, + project: { + name: 'test-repo6', + path_with_namespace: 'awesome-group/test-repo6', + description: 'My Cool Project', + web_url: 'https://example.com/awesome-group/test-repo6', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/awesome-group/test-repo6', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: added_commits, + total_commits_count: 2, + repository: { + name: 'test-repo6', + url: 'https://gitlab.com/awesome-group/test-repo6', + description: 'My Cool Project', + homepage: 'https://gitlab.com/awesome-group/test-repo6', + git_http_url: 'https://gitlab.com/awesome-group/test-repo6.git', + git_ssh_url: 'git@gitlab.com:awesome-group/test-repo6.git', + visibility_level: 20, + }, + }, +}; +export const push_add_event_forked: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 4, + project: { + name: 'test-repo4-forked', + path_with_namespace: 'group1/test-repo4-forked', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo4-forked', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo4-forked', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: added_commits, + total_commits_count: 2, + repository: { + name: 'test-repo4-forked', + url: 'https://gitlab.com/group1/test-repo4-forked', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo4-forked', + git_http_url: 'https://gitlab.com/group1/test-repo4-forked.git', + git_ssh_url: 'git@gitlab.com:group1/test-repo4-forked.git', + visibility_level: 20, + }, + }, +}; + +export const push_remove_event: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 1, + project: { + name: 'test-repo', + path_with_namespace: 'group1/test-repo1', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo1', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo1', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: removed_commits, + total_commits_count: 2, + repository: { + name: 'test-repo', + url: 'https://gitlab.com/group1/test-repo1', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo1', + git_http_url: 'https://gitlab.com/group1/test-repo1.git', + git_ssh_url: 'git@gitlab.com:bgroup1/test-repo1.git', + visibility_level: 20, + }, + }, +}; + +export const push_modif_event: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 1, + project: { + name: 'test-repo', + path_with_namespace: 'group1/test-repo1', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo1', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo1', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: modified_commits, + total_commits_count: 2, + repository: { + name: 'test-repo', + url: 'https://gitlab.com/group1/test-repo1', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo1', + git_http_url: 'https://gitlab.com/group1/test-repo1.git', + git_ssh_url: 'git@gitlab.com:bgroup1/test-repo1.git', + visibility_level: 20, + }, + }, +}; + +/** + * Expected Backstage entities + */ +export const expected_location_entities: MockObject[] = + all_projects_response.map(project => { + const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${project.default_branch}/catalog-info.yaml`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ target: targetUrl, type: 'url' }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }); + +export const expected_added_location_entities: MockObject[] = added_commits.map( + commit => { + const targetUrl = `https://example.com/group1/test-repo1/-/blob/main/${commit.added}`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ target: targetUrl, type: 'url' }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }, +); + +export const expected_removed_location_entities: MockObject[] = + removed_commits.map(commit => { + const targetUrl = `https://example.com/group1/test-repo1/-/blob/main/${commit.removed}`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ target: targetUrl, type: 'url' }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }); + +export const expected_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group1', + 'example.com/team-path': 'group1', + }, + name: 'group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_group_user_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group1', + 'example.com/team-path': 'group1', + }, + name: 'group1', + }, + spec: { + children: [], + members: ['user1'], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_added_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/group-new/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group-new/group1', + 'example.com/team-path': 'group-new/group1', + }, + name: 'group-new-group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_removed_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/group-old/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group-old/group1', + 'example.com/team-path': 'group-old/group1', + }, + name: 'group-old-group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_transformed_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group1', + 'example.com/team-path': 'group1', + }, + name: '1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_single_user_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/JohnDoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/JohnDoe', + 'example.com/user-login': 'https://gitlab.example/john_doe', + }, + name: 'JohnDoe', + }, + spec: { + memberOf: [], + profile: { + displayName: 'John Doe', + email: 'john.doe@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_single_user_removed_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/johndoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/johndoe', + 'example.com/user-login': '', + }, + name: 'johndoe', + }, + spec: { + memberOf: [], + profile: { + displayName: 'John Doe', + email: 'john.doe@company.com', + picture: undefined, + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_full_org_scan_entities: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/JohnDoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/JohnDoe', + 'example.com/user-login': 'https://gitlab.example/john_doe', + }, + name: 'JohnDoe', + }, + spec: { + memberOf: ['group1'], + profile: { + displayName: 'John Doe', + email: 'john.doe@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/JaneDoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/JaneDoe', + 'example.com/user-login': 'https://gitlab.example/jane_doe', + }, + name: 'JaneDoe', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Jane Doe', + email: 'jane.doe@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/MarySmith', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/MarySmith', + 'example.com/user-login': 'https://gitlab.example/mary_smith', + }, + name: 'MarySmith', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Mary Smith', + email: 'mary.smith@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/MarioMario', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/MarioMario', + 'example.com/user-login': 'https://gitlab.example/mario_mario', + }, + name: 'MarioMario', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Mario Mario', + email: 'mario.mario-company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + + expected_group_entity[0], +]; + +export const expected_full_org_scan_entities_saas: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser1', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser1', + 'gitlab.com/user-login': 'https://gitlab.com/testuser1', + 'gitlab.com/saml-external-uid': '51', + }, + name: 'testuser1', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Test User 1', + email: 'testuser1@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser2', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser2', + 'gitlab.com/user-login': 'https://gitlab.com/testuser2', + 'gitlab.com/saml-external-uid': '52', + }, + name: 'testuser2', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Test User 2', + email: 'testuser2@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 99f6e7869b..9dfe8f8525 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -14,130 +14,25 @@ * limitations under the License. */ -import { ConfigReader } from '@backstage/config'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; -import { readGitLabIntegrationConfig } from '@backstage/integration'; import { getVoidLogger } from '@backstage/backend-common'; -import { graphql, rest } from 'msw'; -import { setupServer, SetupServer } from 'msw/node'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { readGitLabIntegrationConfig } from '@backstage/integration'; +import { setupServer } from 'msw/node'; +import { handlers } from '../__testUtils__/handlers'; +import * as mock from '../__testUtils__/mocks'; import { GitLabClient, paginated } from './client'; import { GitLabGroup, GitLabUser } from './types'; -const server = setupServer(); +const server = setupServer(...handlers); setupRequestMockHandlers(server); -const MOCK_CONFIG = readGitLabIntegrationConfig( - new ConfigReader({ - host: 'example.com', - token: 'test-token', - apiBaseUrl: 'https://example.com/api/v4', - baseUrl: 'https://example.com', - }), -); -const FAKE_PAGED_ENDPOINT = `/some-endpoint`; -const FAKE_PAGED_URL = `${MOCK_CONFIG.apiBaseUrl}${FAKE_PAGED_ENDPOINT}`; - -function setupFakeFourPageURL(srv: SetupServer, url: string) { - srv.use( - rest.get(url, (req, res, ctx) => { - const page = req.url.searchParams.get('page'); - const currentPage = page ? Number(page) : 1; - const fakePageCount = 4; - - return res( - // set next page number header if page requested is less than count - ctx.set( - 'x-next-page', - currentPage < fakePageCount ? String(currentPage + 1) : '', - ), - ctx.json([{ someContentOfPage: currentPage }]), - ); - }), - ); -} - -function setupFakeGroupProjectsEndpoint( - srv: SetupServer, - apiBaseUrl: string, - groupID: string, -) { - srv.use( - rest.get(`${apiBaseUrl}/groups/${groupID}/projects`, (_, res, ctx) => { - return res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - description: 'Project One Description', - name: 'Project One', - path: 'project-one', - }, - ]), - ); - }), - ); -} - -function setupFakeInstanceProjectsEndpoint( - srv: SetupServer, - apiBaseUrl: string, -) { - srv.use( - rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => { - return res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - description: 'Project One Description', - name: 'Project One', - path: 'project-one', - }, - { - id: 2, - description: 'Project Two Description', - name: 'Project Two', - path: 'project-two', - }, - ]), - ); - }), - ); -} - -function setupFakeHasFileEndpoint(srv: SetupServer, apiBaseUrl: string) { - srv.use( - rest.head( - `${apiBaseUrl}/projects/group%2Frepo/repository/files/catalog-info.yaml`, - (req, res, ctx) => { - const branch = req.url.searchParams.get('ref'); - if (branch === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); -} - -function setupFakeURLReturnEndpoint(srv: SetupServer, url: string) { - srv.use( - rest.get(url, (req, res, ctx) => { - return res(ctx.json([{ endpoint: req.url.toString() }])); - }), - ); -} - describe('GitLabClient', () => { describe('isSelfManaged', () => { it('returns true if self managed instance', () => { const client = new GitLabClient({ config: readGitLabIntegrationConfig( - new ConfigReader({ - host: 'example.com', - token: 'test-token', - apiBaseUrl: 'https://example.com/api/v4', - }), + new ConfigReader(mock.config_self_managed), ), logger: getVoidLogger(), }); @@ -145,12 +40,7 @@ describe('GitLabClient', () => { }); it('returns false if gitlab.com', () => { const client = new GitLabClient({ - config: readGitLabIntegrationConfig( - new ConfigReader({ - host: 'gitlab.com', - token: 'test-token', - }), - ), + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); expect(client.isSelfManaged()).toBeFalsy(); @@ -158,31 +48,30 @@ describe('GitLabClient', () => { }); describe('pagedRequest', () => { - beforeEach(() => { - // setup fake paginated endpoint with 4 pages each returning one item - setupFakeFourPageURL(server, FAKE_PAGED_URL); - }); - it('should provide immediate items within the page', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const { items } = await client.pagedRequest(FAKE_PAGED_ENDPOINT); + const { items } = await client.pagedRequest(mock.paged_endpoint); // fake page contains exactly one item expect(items).toHaveLength(1); }); it('should request items for a given page number', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const requestedPage = 2; const { items, nextPage } = await client.pagedRequest( - FAKE_PAGED_ENDPOINT, + mock.paged_endpoint, { page: requestedPage, }, @@ -195,12 +84,14 @@ describe('GitLabClient', () => { it('should not have a next page if at the end', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items, nextPage } = await client.pagedRequest( - FAKE_PAGED_ENDPOINT, + mock.paged_endpoint, { page: 4, }, @@ -211,50 +102,50 @@ describe('GitLabClient', () => { }); it('should throw if response is not okay', async () => { - const endpoint = '/unhealthy-endpoint'; - const url = `${MOCK_CONFIG.apiBaseUrl}${endpoint}`; - server.use( - rest.get(url, (_, res, ctx) => { - return res(ctx.status(400), ctx.json({ error: 'some error' })); - }), - ); - const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); // non-200 status code should throw - await expect(() => client.pagedRequest(endpoint)).rejects.toThrow(); + await expect(() => + client.pagedRequest(mock.unhealthy_endpoint), + ).rejects.toThrow(); }); }); + // TODO: review this. Make it so that the number of projects don't match the results describe('listProjects', () => { it('should get projects for a given group', async () => { - setupFakeGroupProjectsEndpoint( - server, - MOCK_CONFIG.apiBaseUrl, - 'test-group', - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const groupProjectsGen = paginated( options => client.listProjects(options), - { group: 'test-group' }, + { group: 1 }, ); + + const expectedProjects = mock.all_projects_response.filter(project => + project.path_with_namespace!.includes('group1/'), + ); + const allItems = []; for await (const item of groupProjectsGen) { allItems.push(item); } - expect(allItems).toHaveLength(1); + expect(allItems).toHaveLength(expectedProjects.length); }); it('should get all projects for an instance', async () => { - setupFakeInstanceProjectsEndpoint(server, MOCK_CONFIG.apiBaseUrl); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); @@ -266,532 +157,172 @@ describe('GitLabClient', () => { for await (const project of instanceProjects) { allProjects.push(project); } - expect(allProjects).toHaveLength(2); + expect(allProjects).toHaveLength(mock.all_projects_response.length); }); }); - it('listUsers gets all users in the instance', async () => { - server.use( - rest.get(`${MOCK_CONFIG.apiBaseUrl}/users`, (_, res, ctx) => - res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - username: 'test1', - name: 'Test Testit', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.example/test1', - created_at: '2023-01-19T07:27:03.333Z', - bio: '', - location: null, - public_email: null, - skype: '', - linkedin: '', - twitter: '', - website_url: '', - organization: null, - job_title: '', - pronouns: null, - bot: false, - work_information: null, - followers: 0, - following: 0, - is_followed: false, - local_time: null, - last_sign_in_at: '2023-01-19T07:27:49.601Z', - confirmed_at: '2023-01-19T07:27:02.905Z', - last_activity_on: '2023-01-19', - email: 'test@example.com', - theme_id: 1, - color_scheme_id: 1, - projects_limit: 100000, - current_sign_in_at: '2023-01-19T09:09:10.676Z', - identities: [], - can_create_group: true, - can_create_project: true, - two_factor_enabled: false, - external: false, - private_profile: false, - commit_email: 'test@example.com', - is_admin: false, - note: '', - }, - ]), + describe('listUsers', () => { + it('listUsers gets all users in the instance', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), ), - ), - ); - const client = new GitLabClient({ - config: MOCK_CONFIG, - logger: getVoidLogger(), + logger: getVoidLogger(), + }); + + const allUsers: GitLabUser[] = []; + for await (const user of paginated( + options => client.listUsers(options), + {}, + )) { + allUsers.push(user); + } + + expect(allUsers).toMatchObject(mock.all_users_response); }); - - const allUsers: GitLabUser[] = []; - for await (const user of paginated( - options => client.listUsers(options), - {}, - )) { - allUsers.push(user); - } - - expect(allUsers).toMatchObject([ - { - id: 1, - username: 'test1', - email: 'test@example.com', - name: 'Test Testit', - state: 'active', - web_url: 'https://gitlab.example/test1', - avatar_url: 'https://secure.gravatar.com/', - }, - ]); }); - it('listGroups gets all groups in the instance', async () => { - server.use( - rest.get(`${MOCK_CONFIG.apiBaseUrl}/groups`, (_, res, ctx) => - res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - web_url: 'https://gitlab.example/groups/group1', - name: 'group1', - path: 'group1', - description: '', - visibility: 'internal', - share_with_group_lock: false, - require_two_factor_authentication: false, - two_factor_grace_period: 48, - project_creation_level: 'developer', - auto_devops_enabled: null, - subgroup_creation_level: 'owner', - emails_disabled: null, - mentions_disabled: null, - lfs_enabled: true, - default_branch_protection: 2, - avatar_url: null, - request_access_enabled: false, - full_name: '8020', - full_path: '8020', - created_at: '2017-06-19T06:42:34.160Z', - parent_id: null, - }, - ]), + describe('listGroups', () => { + it('listGroups gets all groups in the instance', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), ), - ), - ); - const client = new GitLabClient({ - config: MOCK_CONFIG, - logger: getVoidLogger(), + logger: getVoidLogger(), + }); + + const allGroups: GitLabGroup[] = []; + for await (const group of paginated( + options => client.listGroups(options), + {}, + )) { + allGroups.push(group); + } + + expect(allGroups).toMatchObject(mock.all_groups_response); }); - - const allGroups: GitLabGroup[] = []; - for await (const group of paginated( - options => client.listGroups(options), - {}, - )) { - allGroups.push(group); - } - - expect(allGroups).toMatchObject([ - { - id: 1, - name: 'group1', - full_path: '8020', - description: '', - parent_id: null, - }, - ]); }); describe('get gitlab.com users', () => { it('gets all users under group', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: [ - { - user: { - id: 'gid://gitlab/User/1', - username: 'user1', - publicEmail: 'user1@example.com', - name: 'user1', - state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); - const saasMembers = ( - await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + await client.getGroupMembers('saas-multi-user-group', [ + 'DIRECT, DESCENDANTS', + ]) ).items; - const expectedSaasMember = [ - { - id: 1, - username: 'user1', - email: 'user1@example.com', - name: 'user1', - state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', - }, - ]; - expect(saasMembers.length).toEqual(1); - expect(saasMembers).toEqual(expectedSaasMember); + expect(saasMembers.length).toEqual(2); + expect(saasMembers).toEqual(mock.expectedSaasMember); }); - it('gets all users with token without full permissions', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: {}, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); - const saasMembers = ( - await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + await client.getGroupMembers('', ['DIRECT, DESCENDANTS']) ).items; - expect(saasMembers).toEqual([]); }); - it('rejects when GraphQL returns errors', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.errors([ - { message: 'Unexpected end of document', locations: [] }, - ]), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); - await expect(() => - client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']), + client.getGroupMembers('error-group', ['DIRECT, DESCENDANTS']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); it('traverses multi-page results', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: req.variables.endCursor - ? [ - { - user: { - id: 'gid://gitlab/User/1', - username: 'user1', - publicEmail: 'user1@example.com', - name: 'user1', - state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', - }, - }, - ] - : [ - { - user: { - id: 'gid://gitlab/User/2', - username: 'user2', - publicEmail: 'user2@example.com', - name: 'user2', - state: 'active', - webUrl: 'user2.com', - avatarUrl: 'user2', - }, - }, - ], - pageInfo: { - endCursor: req.variables.endCursor ? 'end' : 'next', - hasNextPage: !req.variables.endCursor, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const saasMembers = ( - await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + await client.getGroupMembers('multi-page-saas', ['DIRECT, DESCENDANTS']) ).items; - const expectedSaasMember1 = { - id: 1, - username: 'user1', - email: 'user1@example.com', - name: 'user1', - state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', - }; - - const expectedSaasMember2 = { - id: 2, - username: 'user2', - email: 'user2@example.com', - name: 'user2', - state: 'active', - web_url: 'user2.com', - avatar_url: 'user2', - }; - expect(saasMembers.length).toEqual(2); - expect(saasMembers[0]).toEqual(expectedSaasMember2); - expect(saasMembers[1]).toEqual(expectedSaasMember1); + expect(saasMembers[0]).toEqual(mock.expectedSaasMember[1]); + expect(saasMembers[1]).toEqual(mock.expectedSaasMember[0]); }); }); describe('listDescendantGroups', () => { it('gets all groups under root', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.data({ - group: { - descendantGroups: { - nodes: [ - { - id: 'gid://gitlab/Group/1', - name: 'group1', - description: 'description1', - fullPath: 'path/group1', - parent: { - id: '123', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const saasGroups = (await client.listDescendantGroups('group1')).items; + const allGroups = (await client.listDescendantGroups('group-with-parent')) + .items; - const expectedSaasGroup = [ - { - id: 1, - name: 'group1', - description: 'description1', - full_path: 'path/group1', - parent_id: 123, - }, - ]; - - expect(saasGroups.length).toEqual(1); - expect(saasGroups).toEqual(expectedSaasGroup); + expect(allGroups.length).toEqual(1); + expect(allGroups).toEqual(mock.group_with_parent); }); it('gets all descendant groups with token without full permissions', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.data({ - group: {}, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const saasGroups = (await client.listDescendantGroups('group1')).items; + const allGroups = ( + await client.listDescendantGroups('non-existing-group') + ).items; - expect(saasGroups).toEqual([]); + expect(allGroups).toEqual([]); }); it('rejects when GraphQL returns errors', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.errors([ - { message: 'Unexpected end of document', locations: [] }, - ]), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - await expect(() => client.listDescendantGroups('group1')).rejects.toThrow( + await expect(() => + client.listDescendantGroups('error-group'), + ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); it('traverses multi-page results', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (req, res, ctx) => - res( - ctx.data({ - group: { - descendantGroups: { - nodes: req.variables.endCursor - ? [ - { - id: 'gid://gitlab/Group/1', - name: 'group1', - description: 'description1', - fullPath: 'path/group1', - parent: { - id: '123', - }, - }, - ] - : [ - { - id: 'gid://gitlab/Group/2', - name: 'group2', - description: 'description2', - fullPath: 'path/group2', - parent: { - id: '123', - }, - }, - ], - pageInfo: { - endCursor: req.variables.endCursor ? 'end' : 'next', - hasNextPage: !req.variables.endCursor, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const saasGroups = (await client.listDescendantGroups('root')).items; - const expectedSaasGroup1 = { - id: 1, - name: 'group1', - description: 'description1', - full_path: 'path/group1', - parent_id: 123, - }; - - const expectedSaasGroup2 = { - id: 2, - name: 'group2', - description: 'description2', - full_path: 'path/group2', - parent_id: 123, - }; - expect(saasGroups.length).toEqual(2); - expect(saasGroups[0]).toEqual(expectedSaasGroup2); - expect(saasGroups[1]).toEqual(expectedSaasGroup1); + expect(saasGroups[0]).toEqual(mock.expectedSaasGroup[1]); + expect(saasGroups[1]).toEqual(mock.expectedSaasGroup[0]); }); }); describe('getGroupMembers', () => { it('gets member IDs', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: [ - { - user: { - id: 'gid://gitlab/User/1', - username: 'user1', - publicEmail: 'user1@example.com', - name: 'user1', - state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); @@ -811,96 +342,144 @@ describe('GitLabClient', () => { }); it('gets member IDs with token without full permissions', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: {}, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', ['DIRECT']); + const members = await client.getGroupMembers('non-existing-group', [ + 'DIRECT', + ]); expect(members.items).toEqual([]); }); + // TODO: is this one really necessary? it('rejects when GraphQL returns errors', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.errors([ - { message: 'Unexpected end of document', locations: [] }, - ]), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); await expect(() => - client.getGroupMembers('group1', ['DIRECT']), + client.getGroupMembers('error-group', ['DIRECT']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); it('traverses multi-page results', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: req.variables.endCursor - ? [{ user: { id: 'gid://gitlab/User/2' } }] - : [{ user: { id: 'gid://gitlab/User/1' } }], - pageInfo: { - endCursor: req.variables.endCursor ? 'end' : 'next', - hasNextPage: !req.variables.endCursor, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', ['DIRECT']); + const members = await client.getGroupMembers('multi-page', ['DIRECT']); expect(members.items[0].id).toEqual(1); expect(members.items[1].id).toEqual(2); }); }); + + describe('getGroupById', () => { + it('should return group details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + const group = await client.getGroupById(1); + expect(group).toMatchObject(mock.all_groups_response[0]); + }); + + it('should handle errors when fetching group details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + await expect(() => client.getGroupById(42)).rejects.toThrow( + 'Internal Server Error', + ); + }); + }); + + describe('getProjectById', () => { + it('should return project details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + const project = await client.getProjectById(1); + expect(project).toMatchObject(mock.all_projects_response[0]); + }); + + it('should handle errors when fetching project details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + await expect(() => client.getProjectById(42)).rejects.toThrow( + 'Internal Server Error', + ); + }); + }); + + describe('getUserById', () => { + it('should return user details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + const user = await client.getUserById(1); + expect(user).toMatchObject(mock.all_users_response[0]); + }); + + it('should handle errors when fetching user details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + await expect(() => client.getUserById(42)).rejects.toThrow( + 'Internal Server Error', + ); + }); + }); }); describe('paginated', () => { it('should iterate through the pages until exhausted', async () => { - setupFakeFourPageURL(server, FAKE_PAGED_URL); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const paginatedItems = paginated( - options => client.pagedRequest(FAKE_PAGED_ENDPOINT, options), + options => client.pagedRequest(mock.paged_endpoint, options), {}, ); const allItems = []; @@ -916,25 +495,26 @@ describe('hasFile', () => { let client: GitLabClient; beforeEach(() => { - setupFakeHasFileEndpoint(server, MOCK_CONFIG.apiBaseUrl); client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); }); - it('should not find catalog file', async () => { + it('should find catalog file', async () => { const hasFile = await client.hasFile( - 'group/repo', - 'master', + 'group1/test-repo1', + 'main', 'catalog-info.yaml', ); expect(hasFile).toBe(true); }); - it('should find catalog file', async () => { + it('should not find catalog file', async () => { const hasFile = await client.hasFile( - 'group/repo', + 'group1/test-repo1', 'unknown', 'catalog-info.yaml', ); @@ -943,19 +523,16 @@ describe('hasFile', () => { }); describe('pagedRequest search params', () => { - beforeEach(() => { - // setup fake paginated endpoint with 4 pages each returning one item - setupFakeURLReturnEndpoint(server, FAKE_PAGED_URL); - }); - it('no search params provided', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, ); // fake page contains exactly one item expect(items).toHaveLength(1); @@ -966,12 +543,14 @@ describe('pagedRequest search params', () => { it('defined numeric search params', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, { page: 1, per_page: 50 }, ); // fake page contains exactly one item @@ -985,12 +564,14 @@ describe('pagedRequest search params', () => { it('defined string search params', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, { test: 'value', empty: '' }, ); // fake page contains exactly one item @@ -1004,12 +585,14 @@ describe('pagedRequest search params', () => { it('defined boolean search params', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, { active: true, archived: false }, ); // fake page contains exactly one item diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index cc2b2362cd..b30da0461a 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -24,6 +24,7 @@ import { GitLabDescendantGroupsResponse, GitLabGroup, GitLabGroupMembersResponse, + GitLabProject, GitLabUser, PagedResponse, } from './types'; @@ -80,6 +81,39 @@ export class GitLabClient { return this.pagedRequest(`/projects`, options); } + async getProjectById( + projectId: number, + options?: CommonListOptions, + ): Promise { + // Make the request to the GitLab API + const response = await this.nonPagedRequest( + `/projects/${projectId}`, + options, + ); + + return response; + } + + async getGroupById( + groupId: number, + options?: CommonListOptions, + ): Promise { + // Make the request to the GitLab API + const response = await this.nonPagedRequest(`/groups/${groupId}`, options); + + return response; + } + + async getUserById( + userId: number, + options?: CommonListOptions, + ): Promise { + // Make the request to the GitLab API + const response = await this.nonPagedRequest(`/users/${userId}`, options); + + return response; + } + async listUsers( options?: UserListOptions, ): Promise> { @@ -139,7 +173,6 @@ export class GitLabClient { name description fullPath - visibility parent { id } @@ -174,7 +207,6 @@ export class GitLabClient { name: groupItem.name, description: groupItem.description, full_path: groupItem.fullPath, - visibility: groupItem.visibility, parent_id: Number( groupItem.parent.id.replace(/^gid:\/\/gitlab\/Group\//, ''), ), @@ -325,9 +357,13 @@ export class GitLabClient { options?: CommonListOptions, ): Promise> { const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); + for (const key in options) { - if (options[key] !== undefined && options[key] !== '') { - request.searchParams.append(key, options[key]!.toString()); + if (options.hasOwnProperty(key)) { + const value = options[key]; + if (value !== undefined && value !== '') { + request.searchParams.append(key, value.toString()); + } } } @@ -336,6 +372,7 @@ export class GitLabClient { request.toString(), getGitLabRequestOptions(this.config), ); + if (!response.ok) { throw new Error( `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ @@ -343,6 +380,7 @@ export class GitLabClient { } - ${response.statusText}`, ); } + return response.json().then(items => { const nextPage = response.headers.get('x-next-page'); @@ -352,6 +390,37 @@ export class GitLabClient { } as PagedResponse; }); } + + async nonPagedRequest( + endpoint: string, + options?: CommonListOptions, + ): Promise { + const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); + + for (const key in options) { + if (options.hasOwnProperty(key)) { + const value = options[key]; + if (value !== undefined && value !== '') { + request.searchParams.append(key, value.toString()); + } + } + } + + const response = await fetch( + request.toString(), + getGitLabRequestOptions(this.config), + ); + + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + + return response.json(); + } } /** diff --git a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts index 912e5a0502..d7a6bf76e9 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts @@ -49,10 +49,6 @@ export function defaultGroupEntitiesTransformer( const annotations: { [annotationName: string]: string } = {}; annotations[`${options.providerConfig.host}/team-path`] = group.full_path; - if (group.visibility !== undefined) { - annotations[`${options.providerConfig.host}/visibility`] = - group.visibility; - } const entity: GroupEntity = { apiVersion: 'backstage.io/v1alpha1', diff --git a/plugins/catalog-backend-module-gitlab/src/lib/index.ts b/plugins/catalog-backend-module-gitlab/src/lib/index.ts index 1eee268862..cc82bd6a0b 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/index.ts @@ -14,14 +14,15 @@ * limitations under the License. */ +export { readGitlabConfigs } from '../providers/config'; export { GitLabClient, paginated } from './client'; export type { - GitLabUser, GitLabGroup, GitLabGroupSamlIdentity, GitLabProject, - GitlabProviderConfig, + GitLabUser, GitlabGroupDescription, + GitlabProviderConfig, GroupNameTransformer, GroupNameTransformerOptions, GroupTransformer, @@ -29,4 +30,3 @@ export type { UserTransformer, UserTransformerOptions, } from './types'; -export { readGitlabConfigs } from '../providers/config'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 123515ccf3..b60814ade1 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -34,6 +34,9 @@ export type GitlabProjectForkedFrom = { export type GitLabProject = { id: number; + description?: string; + name?: string; + path?: string; default_branch?: string; archived: boolean; last_activity_at: string; @@ -76,7 +79,6 @@ export type GitLabGroup = { name: string; full_path: string; description?: string; - visibility?: string; parent_id?: number; }; @@ -116,7 +118,6 @@ export type GitLabDescendantGroupsResponse = { name: string; description: string; fullPath: string; - visibility: string; parent: { id: string; }; @@ -180,6 +181,11 @@ export type GitlabProviderConfig = { */ groupPattern: RegExp; + /** + * If true, the provider will also ingest add inherited users to the ingested groups + */ + allowInherited?: boolean; + orgEnabled?: boolean; schedule?: TaskScheduleDefinition; /** @@ -242,3 +248,72 @@ export interface GroupTransformerOptions { providerConfig: GitlabProviderConfig; groupNameTransformer: GroupNameTransformer; } + +/** + * Represents the schema for system hook events related to groups. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseGroupEventsSchema = { + created_at: string; + updated_at: string; + name: string; + path: string; + full_path: string; + group_id: number; +}; + +/** + * Represents the schema for system hook events related to users. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseUserEventsSchema = { + created_at: string; + updated_at: string; + email: string; + name: string; + username: string; + user_id: number; +}; + +/** + * Represents the schema for system hook events related to user memberships. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseMembershipEventsSchema = { + created_at: string; + updated_at: string; + group_name: string; + group_path: string; + group_id: number; + user_username: string; + user_email: string; + user_name: string; + user_id: number; + group_access: string; +}; + +/** + * Represents the schema for system hook events related to projects. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseProjectEventsSchema = { + created_at: string; + updated_at: string; + event_name: string; + name: string; + owner_email: string; + owner_name: string; + owners: { name: string; email: string }[]; + path: string; + path_with_namespace: string; + project_id: number; + project_visibility: string; +}; diff --git a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts index 46cfe403e8..3c93cb98f1 100644 --- a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts @@ -14,15 +14,27 @@ * limitations under the License. */ +import { createServiceFactory } from '@backstage/backend-plugin-api'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { TestEventsService } from '@backstage/plugin-events-backend-test-utils'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; import { Duration } from 'luxon'; -import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider'; import { GitlabDiscoveryEntityProvider } from '../providers'; +import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider'; describe('catalogModuleGitlabDiscoveryEntityProvider', () => { it('should register provider at the catalog extension point', async () => { + const events = new TestEventsService(); + const eventsServiceFactory = createServiceFactory({ + service: eventsServiceRef, + deps: {}, + async factory({}) { + return events; + }, + }); let addedProviders: Array | undefined; let usedSchedule: TaskScheduleDefinition | undefined; @@ -31,6 +43,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => { addedProviders = providers; }, }; + const connection = jest.fn() as unknown as EntityProviderConnection; const runner = jest.fn(); const scheduler = mockServices.scheduler.mock({ createScheduledTaskRunner(schedule) { @@ -68,6 +81,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => { await startTestBackend({ extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], features: [ + eventsServiceFactory(), catalogModuleGitlabDiscoveryEntityProvider(), mockServices.rootConfig.factory({ data: config }), mockServices.logger.factory(), @@ -78,9 +92,17 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => { expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); expect(addedProviders?.length).toEqual(1); - expect(addedProviders?.pop()?.getProviderName()).toEqual( + expect(runner).not.toHaveBeenCalled(); + + const provider = addedProviders!.pop()!; + expect(provider.getProviderName()).toEqual( 'GitlabDiscoveryEntityProvider:test-id', ); - expect(runner).not.toHaveBeenCalled(); + await provider.connect(connection); + expect(events.subscribed).toHaveLength(1); + expect(events.subscribed[0].id).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + expect(runner).toHaveBeenCalledTimes(1); }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts index a1b24e4eea..bba8348a74 100644 --- a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts @@ -19,6 +19,7 @@ import { createBackendModule, } from '@backstage/backend-plugin-api'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; import { GitlabDiscoveryEntityProvider } from '../providers'; /** @@ -26,6 +27,7 @@ import { GitlabDiscoveryEntityProvider } from '../providers'; * * @alpha */ + export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({ pluginId: 'catalog', moduleId: 'gitlab-discovery-entity-provider', @@ -36,14 +38,16 @@ export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({ catalog: catalogProcessingExtensionPoint, logger: coreServices.logger, scheduler: coreServices.scheduler, + events: eventsServiceRef, }, - async init({ config, catalog, logger, scheduler }) { - catalog.addEntityProvider( + async init({ config, catalog, logger, scheduler, events }) { + const gitlabDiscoveryEntityProvider = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, + events, scheduler, - }), - ); + }); + catalog.addEntityProvider(gitlabDiscoveryEntityProvider); }, }); }, diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index 7508b35b84..bd725ee0ce 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -23,10 +23,16 @@ import { import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; -import { rest } from 'msw'; +import { DefaultEventsService } from '@backstage/plugin-events-node'; import { setupServer } from 'msw/node'; +import { handlers } from '../__testUtils__/handlers'; +import * as mock from '../__testUtils__/mocks'; import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; +const server = setupServer(...handlers); +setupRequestMockHandlers(server); +afterEach(() => jest.resetAllMocks()); + class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -42,13 +48,8 @@ class PersistingTaskRunner implements TaskRunner { const logger = getVoidLogger(); -const server = setupServer(); - -describe('GitlabDiscoveryEntityProvider', () => { - setupRequestMockHandlers(server); - afterEach(() => jest.resetAllMocks()); - - it('no provider config', () => { +describe('GitlabDiscoveryEntityProvider - configuration', () => { + it('should not instantiate providers when no config found', () => { const schedule = new PersistingTaskRunner(); const config = new ConfigReader({}); const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { @@ -59,29 +60,46 @@ describe('GitlabDiscoveryEntityProvider', () => { expect(providers).toHaveLength(0); }); - it('single simple discovery config', () => { + it('should fail without schedule nor scheduler', () => { + const config = new ConfigReader(mock.config_single_integration_branch); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('should fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader(mock.config_no_schedule_integration); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id', + ); + }); + it('should throw error when no matching GitLab integration config found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_non_gitlab_host); + + expect(() => { + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow('No gitlab integration found that matches host example.com'); + }); + + it('should instantiate provider with single simple discovery config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader(mock.config_single_integration_branch); const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -93,33 +111,9 @@ describe('GitlabDiscoveryEntityProvider', () => { ); }); - it('multiple discovery configs', () => { + it('should instantiate providers when multiple discovery configs', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - 'second-test': { - host: 'test-gitlab', - group: 'second-group', - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_double_integration); const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -133,29 +127,10 @@ describe('GitlabDiscoveryEntityProvider', () => { 'GitlabDiscoveryEntityProvider:second-test', ); }); - - it('apply full update on scheduled execution', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); +}); +describe('GitlabDiscoveryEntityProvider - refresh', () => { + it('should apply full update on scheduled execution', async () => { + const config = new ConfigReader(mock.config_no_org_integration); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -169,92 +144,28 @@ describe('GitlabDiscoveryEntityProvider', () => { 'GitlabDiscoveryEntityProvider:test-id', ); - server.use( - rest.get( - `https://api.gitlab.example/api/v4/groups/test-group/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); - await provider.connect(entityProviderConnection); const taskDef = schedule.getTasks()[0]; expect(taskDef.id).toEqual('GitlabDiscoveryEntityProvider:test-id:refresh'); await (taskDef.fn as () => Promise)(); - const url = `https://api.gitlab.example/test-group/test-repo/-/blob/master/catalog-info.yaml`; - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': `url:${url}`, - 'backstage.io/managed-by-origin-location': `url:${url}`, - }, - name: 'generated-cd37bf72a2fe92603f4255d9f49c6c1ead746a48', - }, - spec: { - presence: 'optional', - target: `${url}`, - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ]; - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: expectedEntities, + entities: mock.expected_location_entities.filter( + entity => + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), }); }); it('should filter found projects based on a provided project pattern', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - projectPattern: 'john/', - }, - }, - }, - }, - }); + const config = new ConfigReader( + mock.config_single_integration_project_pattern, + ); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -265,50 +176,10 @@ describe('GitlabDiscoveryEntityProvider', () => { schedule, })[0]; - server.use( - rest.get( - `https://api.gitlab.example/api/v4/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - { - id: 124, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/john/example', - path_with_namespace: 'john/example', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); + const projectPattern = + mock.config_single_integration_project_pattern.catalog.providers.gitlab[ + 'test-id' + ].projectPattern; await provider.connect(entityProviderConnection); @@ -316,55 +187,20 @@ describe('GitlabDiscoveryEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - 'backstage.io/managed-by-origin-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - }, - name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9', - }, - spec: { - presence: 'optional', - target: - 'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ], + entities: mock.expected_location_entities.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(projectPattern) && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), }); }); it('should filter fork projects', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - skipForkedRepos: true, - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_single_integration_skip_forks); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -375,216 +211,26 @@ describe('GitlabDiscoveryEntityProvider', () => { schedule, })[0]; - server.use( - rest.get( - `https://api.gitlab.example/api/v4/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - forked_from_project: { - id: 13083, - }, - }, - { - id: 124, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/john/example', - path_with_namespace: 'john/example', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); - await provider.connect(entityProviderConnection); await provider.refresh(logger); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - 'backstage.io/managed-by-origin-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - }, - name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9', - }, - spec: { - presence: 'optional', - target: - 'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ], + entities: mock.expected_location_entities.filter( + entity => + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('forked') && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), }); }); - it('fail without schedule and scheduler', () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); - - expect(() => - GitlabDiscoveryEntityProvider.fromConfig(config, { - logger, - }), - ).toThrow('Either schedule or scheduler must be provided'); - }); - - it('fail with scheduler but no schedule config', () => { - const scheduler = { - createScheduledTaskRunner: (_: any) => jest.fn(), - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); - - expect(() => - GitlabDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ).toThrow( - 'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id', - ); - }); - - it('single simple provider config with schedule in config', async () => { - const schedule = new PersistingTaskRunner(); - const scheduler = { - createScheduledTaskRunner: (_: any) => schedule, - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - schedule: { - frequency: 'PT30M', - timeout: 'PT3M', - }, - }, - }, - }, - }, - }); - const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }); - - expect(providers).toHaveLength(1); - expect(providers[0].getProviderName()).toEqual( - 'GitlabDiscoveryEntityProvider:test-id', - ); - }); - it('should filter found projects based on the branch', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - branch: 'test', - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_single_integration_branch); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -595,47 +241,9 @@ describe('GitlabDiscoveryEntityProvider', () => { schedule, })[0]; - server.use( - rest.get( - `https://api.gitlab.example/api/v4/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - { - id: 124, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/john/example', - path_with_namespace: 'john/example', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (_, res, ctx) => { - return res(ctx.status(404, 'Not Found')); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'test') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); + const configured_branch = + mock.config_single_integration_branch.catalog.providers.gitlab['test-id'] + .branch; await provider.connect(entityProviderConnection); @@ -643,30 +251,229 @@ describe('GitlabDiscoveryEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml', - 'backstage.io/managed-by-origin-location': - 'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml', - }, - name: 'generated-232185d858fee049986d202c10316d634e76a3d1', - }, - spec: { - presence: 'optional', - target: - 'https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml', - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ], + entities: mock.expected_location_entities.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(configured_branch) && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), + }); + }); + + it('should only include projects with fallback branch', async () => { + const config = new ConfigReader(mock.config_fallbackBranch_branch); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const configured_branch = + mock.config_fallbackBranch_branch.catalog.providers.gitlab['test-id'] + .fallbackBranch; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(configured_branch) && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), + }); + }); + + it('should ignore projects outside group scope', async () => { + const config = new ConfigReader(mock.config_single_integration_group); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const configured_group = + mock.config_single_integration_group.catalog.providers.gitlab['test-id'] + .group; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities.filter(entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(configured_group), + ), }); }); }); +describe('GitlabDiscoveryEntityProvider - events', () => { + it('should ignore push event if project is forked', async () => { + const config = new ConfigReader(mock.config_single_integration_skip_forks); + + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event_forked); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it(`should skip refresh and mutation when project pattern doesn't match`, async () => { + const config = new ConfigReader(mock.config_unmatched_project_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should ignore projects outside group scope', async () => { + const config = new ConfigReader(mock.config_single_integration_group); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event_unmatched_group); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + it('should apply delta mutations on added files from push event', async () => { + const config = new ConfigReader(mock.config_single_integration_branch); + + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_added_location_entities, + removed: [], + }); + }); + + it('should apply delta mutations on removed files from push event', async () => { + const config = new ConfigReader(mock.config_single_integration_branch); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_remove_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: mock.expected_removed_location_entities, + }); + }); + + it('should call refresh on added files from push event', async () => { + const config = new ConfigReader(mock.config_single_integration_branch); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + const url = `https://example.com/group1/test-repo1`; + + await events.publish(mock.push_modif_event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.refresh).toHaveBeenCalledWith({ + keys: [ + `url:${url}/-/tree/main/catalog-info.yaml`, + `url:${url}/-/blob/main/catalog-info.yaml`, + ], + }); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + // EventSupportChange: stop add tests >>> +}); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index ae8fe4e12c..653b12060e 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -17,12 +17,15 @@ import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, - LocationSpec, locationSpecToLocationEntity, } from '@backstage/plugin-catalog-node'; +import { EventsService } from '@backstage/plugin-events-node'; +import { WebhookProjectSchema, WebhookPushEventSchema } from '@gitbeaker/rest'; import * as uuid from 'uuid'; import { GitLabClient, @@ -33,26 +36,37 @@ import { } from '../lib'; import { LoggerService } from '@backstage/backend-plugin-api'; +import * as path from 'path'; + +const TOPIC_REPO_PUSH = 'gitlab.push'; + type Result = { scanned: number; matches: GitLabProject[]; }; /** - * Discovers entity definition files in the groups of a Gitlab instance. + * Discovers catalog files located in your GitLab instance. + * The provider will search your GitLab instance's projects and register catalog files matching the configured path + * as Location entity and via following processing steps add all contained catalog entities. + * This can be useful as an alternative to static locations or manually adding things to the catalog. + * * @public */ +// <<< EventSupportChange: implemented EventSubscriber interface export class GitlabDiscoveryEntityProvider implements EntityProvider { private readonly config: GitlabProviderConfig; private readonly integration: GitLabIntegration; private readonly logger: LoggerService; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; + private readonly events?: EventsService; static fromConfig( config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; }, @@ -92,13 +106,20 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }), ); }); + return providers; } + /** + * Constructs a GitlabDiscoveryEntityProvider instance. + * + * @param options - Configuration options including config, integration, logger, and taskRunner. + */ private constructor(options: { config: GitlabProviderConfig; integration: GitLabIntegration; logger: LoggerService; + events?: EventsService; taskRunner: TaskRunner; }) { this.config = options.config; @@ -107,6 +128,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { target: this.getProviderName(), }); this.scheduleFn = this.createScheduleFn(options.taskRunner); + this.events = options.events; } getProviderName(): string { @@ -116,8 +138,28 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { async connect(connection: EntityProviderConnection): Promise { this.connection = connection; await this.scheduleFn(); + + if (this.events) { + await this.events.subscribe({ + id: this.getProviderName(), + topics: [TOPIC_REPO_PUSH], + onEvent: async params => { + if (params.topic !== TOPIC_REPO_PUSH) { + return; + } + + await this.onRepoPush(params.eventPayload as WebhookPushEventSchema); + }, + }); + } } + /** + * Creates a scheduled task runner for refreshing the entity provider. + * + * @param taskRunner - The task runner instance. + * @returns The scheduled function. + */ private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; @@ -143,13 +185,17 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }; } + /** + * Performs a full scan on the GitLab instance searching for locations to be ingested + * + * @param logger - The logger instance for logging. + */ async refresh(logger: LoggerService): Promise { if (!this.connection) { throw new Error( `Gitlab discovery connection not initialized for ${this.getProviderName()}`, ); } - const client = new GitLabClient({ config: this.integration.config, logger: logger, @@ -171,38 +217,8 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }; for await (const project of projects) { - if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) { - continue; - } - - res.scanned++; - - if ( - this.config.skipForkedRepos && - project.hasOwnProperty('forked_from_project') - ) { - continue; - } - - if ( - !this.config.branch && - this.config.fallbackBranch === '*' && - project.default_branch === undefined - ) { - continue; - } - - const project_branch = - this.config.branch ?? - project.default_branch ?? - this.config.fallbackBranch; - - const projectHasFile: boolean = await client.hasFile( - project.path_with_namespace ?? '', - project_branch, - this.config.catalogFile, - ); - if (projectHasFile) { + if (await this.shouldProcessProject(project, client)) { + res.scanned++; res.matches.push(project); } } @@ -222,10 +238,255 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { this.config.branch ?? project.default_branch ?? this.config.fallbackBranch; + return { type: 'url', target: `${project.web_url}/-/blob/${project_branch}/${this.config.catalogFile}`, presence: 'optional', }; } + + /** + * Handles the "gitlab.push" event. + * + * @param event - The push event payload. + */ + private async onRepoPush(event: WebhookPushEventSchema): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + this.logger.info( + `Received push event for ${event.project.path_with_namespace}`, + ); + + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + const project = await client.getProjectById(event.project_id); + + if (!project) { + this.logger.debug( + `Ignoring push event for ${event.project.path_with_namespace}`, + ); + + return; + } + + if (!(await this.shouldProcessProject(project, client))) { + this.logger.debug(`Skipping event ${event.project.path_with_namespace}`); + return; + } + + // Get array of added, removed or modified files from the push event + const added = this.getFilesMatchingConfig( + event, + 'added', + this.config.catalogFile, + ); + const removed = this.getFilesMatchingConfig( + event, + 'removed', + this.config.catalogFile, + ); + const modified = this.getFilesMatchingConfig( + event, + 'modified', + this.config.catalogFile, + ); + + // Modified files will be scheduled to a refresh + const addedEntities = this.createLocationSpecCommitedFiles( + event.project, + added, + ); + + const removedEntities = this.createLocationSpecCommitedFiles( + event.project, + removed, + ); + + if (addedEntities.length > 0 || removedEntities.length > 0) { + await this.connection.applyMutation({ + type: 'delta', + added: this.toDeferredEntities( + addedEntities.map(entity => entity.target), + ), + removed: this.toDeferredEntities( + removedEntities.map(entity => entity.target), + ), + }); + } + + if (modified.length > 0) { + const projectBranch = + this.config.branch ?? + event.project.default_branch ?? + this.config.fallbackBranch; + + // scheduling a refresh to both tree and blob (https://git-scm.com/book/en/v2/Git-Internals-Git-Objects) + await this.connection.refresh({ + keys: [ + ...modified.map( + filePath => + `url:${event.project.web_url}/-/tree/${projectBranch}/${filePath}`, + ), + ...modified.map( + filePath => + `url:${event.project.web_url}/-/blob/${projectBranch}/${filePath}`, + ), + ], + }); + } + + this.logger.info( + `Processed GitLab push event from ${event.project.web_url}: added ${added.length} - removed ${removed.length} - modified ${modified.length}`, + ); + } + + /** + * Gets files matching the specified commit action and catalog file name. + * + * @param event - The push event payload. + * @param action - The action type ('added', 'removed', or 'modified'). + * @param catalogFile - The catalog file name. + * @returns An array of file paths. + */ + private getFilesMatchingConfig( + event: WebhookPushEventSchema, + action: 'added' | 'removed' | 'modified', + catalogFile: string, + ): string[] { + if (!event.commits) { + return []; + } + + const matchingFiles = event.commits.flatMap((element: any) => + element[action].filter( + (file: string) => path.basename(file) === catalogFile, + ), + ); + + if (matchingFiles.length === 0) { + this.logger.debug( + `No files matching '${catalogFile}' found in the commits.`, + ); + } + + return matchingFiles; + } + + /** + * Creates Backstage location specs for committed files. + * + * @param project - The GitLab project information. + * @param addedFiles - The array of added file paths. + * @returns An array of location specs. + */ + private createLocationSpecCommitedFiles( + project: WebhookProjectSchema, + addedFiles: string[], + ): LocationSpec[] { + const projectBranch = + this.config.branch ?? + project.default_branch ?? + this.config.fallbackBranch; + + // Filter added files that match the catalog file pattern + const matchingFiles = addedFiles.filter( + file => path.basename(file) === this.config.catalogFile, + ); + + // Create a location spec for each matching file + const locationSpecs: LocationSpec[] = matchingFiles.map(file => ({ + type: 'url', + target: `${project.web_url}/-/blob/${projectBranch}/${file}`, + presence: 'optional', + })); + + return locationSpecs; + } + + /** + * Converts a target URL to a LocationSpec object. + * + * @param {string} target - The target URL to be converted. + * @returns {LocationSpec} The LocationSpec object representing the URL. + */ + private toLocationSpec(target: string): LocationSpec { + return { + type: 'url', + target: target, + presence: 'optional', + }; + } + + private toDeferredEntities(targets: string[]): DeferredEntity[] { + return targets + .map(target => { + const location = this.toLocationSpec(target); + + return locationSpecToLocationEntity({ location }); + }) + .map(entity => { + return { + locationKey: this.getProviderName(), + entity: entity, + }; + }); + } + + private async shouldProcessProject( + project: GitLabProject, + client: GitLabClient, + ): Promise { + if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it does not match the project pattern ${this.config.projectPattern}.`, + ); + return false; + } + + if ( + this.config.group && + !project.path_with_namespace!.startsWith(`${this.config.group}/`) + ) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it does not match the group pattern ${this.config.group}.`, + ); + return false; + } + + if ( + this.config.skipForkedRepos && + project.hasOwnProperty('forked_from_project') + ) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it is a forked project.`, + ); + return false; + } + + const customFallbackBranch = + this.config.fallbackBranch !== 'master' + ? this.config.fallbackBranch + : undefined; + + const project_branch = + this.config.branch ?? + customFallbackBranch ?? + project.default_branch ?? + this.config.fallbackBranch; + + const hasFile = await client.hasFile( + project.path_with_namespace ?? '', + project_branch, + this.config.catalogFile, + ); + + return hasFile; + } } diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 8dffdf1d7b..a65b0a1bc2 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -23,10 +23,17 @@ import { import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; -import { graphql, rest } from 'msw'; +import { DefaultEventsService } from '@backstage/plugin-events-node'; import { setupServer } from 'msw/node'; +import { handlers } from '../__testUtils__/handlers'; +import * as mock from '../__testUtils__/mocks'; +import { GroupNameTransformerOptions } from '../lib/types'; import { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider'; +const server = setupServer(...handlers); +setupRequestMockHandlers(server); +afterEach(() => jest.resetAllMocks()); + class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -42,13 +49,8 @@ class PersistingTaskRunner implements TaskRunner { const logger = getVoidLogger(); -const server = setupServer(); - -describe('GitlabOrgDiscoveryEntityProvider', () => { - setupRequestMockHandlers(server); - afterEach(() => jest.resetAllMocks()); - - it('no provider config', () => { +describe('GitlabOrgDiscoveryEntityProvider - configuration', () => { + it('should not instantiate providers when no config found', () => { const schedule = new PersistingTaskRunner(); const config = new ConfigReader({}); const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { @@ -59,61 +61,89 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { expect(providers).toHaveLength(0); }); - it('single simple discovery config with org disabled', () => { + it('should throw error when no matching GitLab integration config found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); - const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - schedule, - }); + const config = new ConfigReader(mock.config_non_gitlab_host); - expect(providers).toHaveLength(0); + expect(() => { + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow('No gitlab integration found that matches host example.com'); }); - it('single simple discovery config with org enabled', () => { + it('should throw error when org configuration not found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - }, - }, - }, + const config = new ConfigReader(mock.config_no_org_integration); + + expect(() => { + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow('Org not enabled for test-id'); + }); + + it('should throw error when saas without group configuration', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader(mock.config_saas_no_group); + + expect(() => { + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow( + `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`, + ); + }); + it('should fail without schedule and scheduler', () => { + const config = new ConfigReader(mock.config_org_integration_saas); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('should fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader(mock.config_org_integration_saas); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:test-id', + ); + }); + + it('should instantiate provider with single simple saas config', async () => { + const schedule = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + const config = new ConfigReader(mock.config_org_integration_saas_sched); + const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + }); + + it('should instantiate provider when org enabled', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader(mock.config_org_integration); const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -125,35 +155,9 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); }); - it('multiple discovery configs', () => { + it('should instantiate providers when multiple valid provider configs', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - 'second-test': { - host: 'test-gitlab', - group: 'second-group', - orgEnabled: true, - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_org_double_integration); const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -167,29 +171,11 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { 'GitlabOrgDiscoveryEntityProvider:second-test', ); }); +}); - it('apply full update on scheduled execution', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - orgEnabled: true, - }, - }, - }, - }, - }); +describe('GitlabOrgDiscoveryEntityProvider - refresh', () => { + it('should apply full update on scheduled execution', async () => { + const config = new ConfigReader(mock.config_org_integration); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -203,185 +189,6 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { 'GitlabOrgDiscoveryEntityProvider:test-id', ); - server.use( - rest.get( - `https://api.gitlab.example/api/v4/groups/test-group/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.get(`https://api.gitlab.example/api/v4/users`, (_req, res, ctx) => { - const response = [ - { - id: 1, - username: 'test1', - name: 'Test Testit', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.example/test1', - created_at: '2023-01-19T07:27:03.333Z', - bio: '', - location: null, - public_email: null, - skype: '', - linkedin: '', - twitter: '', - website_url: '', - organization: null, - job_title: '', - pronouns: null, - bot: false, - work_information: null, - followers: 0, - following: 0, - is_followed: false, - local_time: null, - last_sign_in_at: '2023-01-19T07:27:49.601Z', - confirmed_at: '2023-01-19T07:27:02.905Z', - last_activity_on: '2023-01-19', - email: 'test@example.com', - theme_id: 1, - color_scheme_id: 1, - projects_limit: 100000, - current_sign_in_at: '2023-01-19T09:09:10.676Z', - identities: [], - can_create_group: true, - can_create_project: true, - two_factor_enabled: false, - external: false, - private_profile: false, - commit_email: 'test@example.com', - is_admin: false, - note: '', - }, - { - id: 2, - username: 'test2', - name: '', - state: 'active', - avatar_url: '', - web_url: 'https://gitlab.example/test2', - created_at: '2023-01-19T07:27:03.333Z', - bio: '', - location: null, - public_email: null, - skype: '', - linkedin: '', - twitter: '', - website_url: '', - organization: null, - job_title: '', - pronouns: null, - bot: false, - work_information: null, - followers: 0, - following: 0, - is_followed: false, - local_time: null, - last_sign_in_at: '2023-01-19T07:27:49.601Z', - confirmed_at: '2023-01-19T07:27:02.905Z', - last_activity_on: '2023-01-19', - email: 'test@example.com', - theme_id: 1, - color_scheme_id: 1, - projects_limit: 100000, - current_sign_in_at: '2023-01-19T09:09:10.676Z', - identities: [], - can_create_group: true, - can_create_project: true, - two_factor_enabled: false, - external: false, - private_profile: false, - commit_email: 'test@example.com', - is_admin: false, - note: '', - }, - ]; - return res(ctx.json(response)); - }), - rest.get(`https://api.gitlab.example/api/v4/groups`, (_req, res, ctx) => { - const response = [ - { - id: 1, - web_url: 'https://gitlab.example/groups/group1', - name: 'group1', - path: 'group1', - description: '', - visibility: 'internal', - share_with_group_lock: false, - require_two_factor_authentication: false, - two_factor_grace_period: 48, - project_creation_level: 'developer', - auto_devops_enabled: null, - subgroup_creation_level: 'owner', - emails_disabled: null, - mentions_disabled: null, - lfs_enabled: true, - default_branch_protection: 2, - avatar_url: null, - request_access_enabled: false, - full_name: '8020', - full_path: '8020', - created_at: '2017-06-19T06:42:34.160Z', - parent_id: null, - }, - { - id: 2, - web_url: 'https://gitlab.example/groups/group1/group2', - name: 'group2', - path: 'group1/group2', - description: 'Group2', - visibility: 'internal', - share_with_group_lock: false, - require_two_factor_authentication: false, - two_factor_grace_period: 48, - project_creation_level: 'developer', - auto_devops_enabled: null, - subgroup_creation_level: 'owner', - emails_disabled: null, - mentions_disabled: null, - lfs_enabled: true, - request_access_enabled: false, - full_name: 'group2', - full_path: 'group1/group2', - created_at: '2017-12-07T13:20:40.675Z', - parent_id: null, - }, - ]; - return res(ctx.json(response)); - }), - graphql - .link('https://test-gitlab/api/graphql') - .operation(async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: - req.variables.group === 'group1/group2' - ? [{ user: { id: 'gid://gitlab/User/1' } }] - : [], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); - await provider.connect(entityProviderConnection); const taskDef = schedule.getTasks()[0]; @@ -390,115 +197,15 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); await (taskDef.fn as () => Promise)(); - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://test-gitlab/test1', - 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/test1', - 'test-gitlab/user-login': 'https://gitlab.example/test1', - }, - name: 'test1', - }, - spec: { - memberOf: ['group1-group2'], - profile: { - displayName: 'Test Testit', - email: 'test@example.com', - picture: 'https://secure.gravatar.com/', - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://test-gitlab/test2', - 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/test2', - 'test-gitlab/user-login': 'https://gitlab.example/test2', - }, - name: 'test2', - }, - spec: { - memberOf: [], - profile: { - displayName: undefined, - email: 'test@example.com', - picture: undefined, - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://test-gitlab/group1/group2', - 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/group1/group2', - 'test-gitlab/team-path': 'group1/group2', - 'test-gitlab/visibility': 'internal', - }, - description: 'Group2', - name: 'group1-group2', - }, - spec: { - children: [], - profile: { - displayName: 'group2', - }, - type: 'team', - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - ]; - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: expectedEntities, + entities: mock.expected_full_org_scan_entities, }); }); - it('apply full update on scheduled execution for gitlab.com', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'gitlab.com', - apiBaseUrl: 'https://gitlab.com/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'gitlab.com', - group: 'group1', - orgEnabled: true, - }, - }, - }, - }, - }); + it('should exclude inactive user', async () => { + const config = new ConfigReader(mock.config_org_integration); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -512,126 +219,70 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { 'GitlabOrgDiscoveryEntityProvider:test-id', ); - server.use( - graphql - .link('https://gitlab.com/api/graphql') - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.data({ - group: { - descendantGroups: { - nodes: [ - { - id: 'gid://gitlab/Group/456', - name: 'group2', - description: 'Group2', - fullPath: 'group1/group2', - visibility: 'private', - parent: { - id: 'gid://gitlab/Group/123', - }, - }, - { - id: 'gid://gitlab/Group/789', - name: 'group3', - description: 'Group3', - fullPath: 'group1/group3', - visibility: 'public', - parent: { - id: 'gid://gitlab/Group/123', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - graphql - .link('https://gitlab.com/api/graphql') - .query('getGroupMembers', async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: - req.variables.group === 'group1/group2' - ? [{ user: { id: 'gid://gitlab/User/12' } }] - : [{ user: { id: 'gid://gitlab/User/34' } }], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - rest.get( - `https://gitlab.com/api/v4/groups/group1/members/all`, - (_req, res, ctx) => { - const response = [ - { - access_level: 30, - created_at: '2023-07-17T08:58:34.984Z', - expires_at: null, - id: 12, - username: 'testuser1', - name: 'Test User 1', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.com/testuser1', - email: 'testuser1@example.com', - group_saml_identity: { - provider: 'group_saml', - extern_uid: '51', - saml_provider_id: 1, - }, - is_using_seat: true, - membership_state: 'active', - }, - { - access_level: 30, - created_at: '2023-07-19T08:58:34.984Z', - expires_at: null, - id: 34, - username: 'testuser2', - name: 'Test User 2', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.com/testuser2', - email: 'testuser2@example.com', - group_saml_identity: { - provider: 'group_saml', - extern_uid: '52', - saml_provider_id: 1, - }, - is_using_seat: true, - membership_state: 'active', - }, - { - access_level: 50, - created_at: '2023-07-15T08:58:34.984Z', - expires_at: '2023-10-26', - id: 54, - username: 'group_100_bot_23dc8057bef66e05181f39be4652577c', - name: 'Token Bot', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: - 'https://gitlab.com/group_100_bot_23dc8057bef66e05181f39be4652577c', - group_saml_identity: null, - is_using_seat: false, - membership_state: 'active', - }, - ]; - return res(ctx.json(response)); - }, - ), + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', + ); + await (taskDef.fn as () => Promise)(); + const userEntities = mock.expected_full_org_scan_entities.filter( + entity => entity.entity.kind === 'User', + ); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(userEntities).not.toHaveLength(mock.all_users_response.length); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_full_org_scan_entities, + }); + }); + + it('should exclude user when email or username does not match userPattern', async () => { + const config = new ConfigReader(mock.config_userPattern_integration); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + + await (taskDef.fn as () => Promise)(); + const userEntities = mock.expected_full_org_scan_entities.filter( + element => element.entity.metadata.name !== 'MarioMario', + ); // filter out user with non matched e-mail + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(userEntities).not.toHaveLength(mock.all_users_response.length); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: userEntities, + }); + }); + + it('should apply full update on scheduled execution for gitlab.com', async () => { + const config = new ConfigReader(mock.config_org_integration_saas); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', ); await provider.connect(entityProviderConnection); @@ -642,263 +293,340 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); await (taskDef.fn as () => Promise)(); - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/testuser1', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/testuser1', - 'gitlab.com/user-login': 'https://gitlab.com/testuser1', - 'gitlab.com/saml-external-uid': '51', - }, - name: 'testuser1', - }, - spec: { - memberOf: ['group2'], - profile: { - displayName: 'Test User 1', - email: 'testuser1@example.com', - picture: 'https://secure.gravatar.com/', - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/testuser2', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/testuser2', - 'gitlab.com/user-login': 'https://gitlab.com/testuser2', - 'gitlab.com/saml-external-uid': '52', - }, - name: 'testuser2', - }, - spec: { - memberOf: ['group3'], - profile: { - displayName: 'Test User 2', - email: 'testuser2@example.com', - picture: 'https://secure.gravatar.com/', - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/group1/group2', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/group1/group2', - 'gitlab.com/team-path': 'group1/group2', - 'gitlab.com/visibility': 'private', - }, - description: 'Group2', - name: 'group2', - }, - spec: { - children: [], - profile: { - displayName: 'group2', - }, - type: 'team', - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/group1/group3', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/group1/group3', - 'gitlab.com/team-path': 'group1/group3', - 'gitlab.com/visibility': 'public', - }, - description: 'Group3', - name: 'group3', - }, - spec: { - children: [], - profile: { - displayName: 'group3', - }, - type: 'team', - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - ]; - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: expectedEntities, + entities: mock.expected_full_org_scan_entities_saas, + }); + }); +}); + +describe('GitlabOrgDiscoveryEntityProvider with events support', () => { + it('should ignore gitlab.group_destroy event when group pattern does not match', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_destroy_event_unmatched); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should apply a delta mutation if gitlab.group_destroy event received and defaultGroupTransformation in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_destroy_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: mock.expected_group_entity, + }); + }); + + it('should ignore gitlab.group_create event when group pattern does not match', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + await events.publish(mock.group_create_event_unmatched); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + it('should apply a delta mutation if gitlab.group_create event received and defaultGroupTransformation in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + await events.publish(mock.group_create_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_group_entity, + removed: [], + }); + }); + + it('should apply a delta mutation if gitlab.group_create event received and user/group transformations in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + function customGroupNameTransformer( + options: GroupNameTransformerOptions, + ): string { + return `${options.group.id}`; + } + + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + groupNameTransformer: customGroupNameTransformer, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_create_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_transformed_group_entity, + removed: [], + }); + }); + + it('should apply a delta mutation if gitlab.group_rename event received and defaultGroupTransformation in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_rename_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_added_group_entity, + removed: mock.expected_removed_group_entity, + }); + }); + + it('should apply a delta mutation if gitlab.user_create event received', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_create_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_single_user_entity, + removed: [], + }); + }); + + it('should apply a delta mutation if gitlab.user_destroy event received', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_destroy_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: mock.expected_single_user_removed_entity, + }); + }); + + it('should apply a delta mutation if gitlab.user_add_to_group event received and defaultGroupTransformer', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_add_to_group_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_group_user_entity, + removed: [], + }); + }); + + it('should ignore a delta mutation if gitlab.user_add_to_group event received and group outside scope', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_add_to_group_event_mismatched); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should apply a delta mutation if gitlab.user_remove_from_group event received and customGroupTransformer', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + function customGroupNameTransformer( + options: GroupNameTransformerOptions, + ): string { + return `${options.group.id}`; + } + + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + groupNameTransformer: customGroupNameTransformer, + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_remove_from_group_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_transformed_group_entity, + removed: [], }); }); - - it('fail without schedule and scheduler', () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - }, - }, - }, - }); - - expect(() => - GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - }), - ).toThrow('Either schedule or scheduler must be provided'); - }); - - it('fail with scheduler but no schedule config', () => { - const scheduler = { - createScheduledTaskRunner: (_: any) => jest.fn(), - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - }, - }, - }, - }); - - expect(() => - GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ).toThrow( - 'No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:test-id', - ); - }); - - it('fail with scheduler but no group config when host is gitlab.com', () => { - const scheduler = { - createScheduledTaskRunner: (_: any) => jest.fn(), - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'gitlab.com', - apiBaseUrl: 'https://gitlab.com/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'gitlab.com', - orgEnabled: true, - }, - }, - }, - }, - }); - - expect(() => - GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ).toThrow( - `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`, - ); - }); - - it('single simple provider config with schedule in config', async () => { - const schedule = new PersistingTaskRunner(); - const scheduler = { - createScheduledTaskRunner: (_: any) => schedule, - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - schedule: { - frequency: 'PT30M', - timeout: 'PT3M', - }, - }, - }, - }, - }, - }); - const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }); - - expect(providers).toHaveLength(1); - expect(providers[0].getProviderName()).toEqual( - 'GitlabOrgDiscoveryEntityProvider:test-id', - ); - }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 90a7a14b6b..aba888aea1 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -25,6 +25,7 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; +import { EventsService } from '@backstage/plugin-events-node'; import { merge } from 'lodash'; import * as uuid from 'uuid'; import { LoggerService } from '@backstage/backend-plugin-api'; @@ -35,21 +36,24 @@ import { paginated, readGitlabConfigs, } from '../lib'; -import { - GitLabGroup, - GitLabUser, - GroupNameTransformer, - GroupTransformer as GroupEntitiesTransformer, - PagedResponse, - UserTransformer, -} from '../lib/types'; import { defaultGroupEntitiesTransformer, defaultGroupNameTransformer, defaultUserTransformer, } from '../lib/defaultTransformers'; +import { + GitLabGroup, + GitLabUser, + GroupTransformer as GroupEntitiesTransformer, + GroupNameTransformer, + PagedResponse, + SystemHookBaseGroupEventsSchema, + SystemHookBaseMembershipEventsSchema, + SystemHookBaseUserEventsSchema, + UserTransformer, +} from '../lib/types'; -type Result = { +type UserResult = { scanned: number; matches: GitLabUser[]; }; @@ -59,6 +63,36 @@ type GroupResult = { matches: GitLabGroup[]; }; +type SystemHookGroupCreateOrDestroyEventSchema = + SystemHookBaseGroupEventsSchema & { + event_name: 'group_create' | 'group_destroy'; + }; + +type SystemHookGroupRenameEventSchema = SystemHookBaseGroupEventsSchema & { + event_name: 'group_rename'; + old_path: string; + old_full_path: string; +}; + +type SystemHookUserCreateOrDestroyEventSchema = + SystemHookBaseUserEventsSchema & { + event_name: 'user_create' | 'user_destroy'; + }; + +type SystemHookCreateOrDestroyMembershipEventsSchema = + SystemHookBaseMembershipEventsSchema & { + event_name: 'user_add_to_group' | 'user_remove_from_group'; + }; + +// System level events +const TOPIC_GROUP_CREATE = 'gitlab.group_create'; +const TOPIC_GROUP_DESTROY = 'gitlab.group_destroy'; +const TOPIC_GROUP_RENAME = 'gitlab.group_rename'; +const TOPIC_USER_CREATE = 'gitlab.user_create'; +const TOPIC_USER_DESTROY = 'gitlab.user_destroy'; +const TOPIC_USER_ADD_GROUP = 'gitlab.user_add_to_group'; +const TOPIC_USER_REMOVE_GROUP = 'gitlab.user_remove_from_group'; + /** * Discovers users and groups from a Gitlab instance. * @public @@ -67,6 +101,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private readonly config: GitlabProviderConfig; private readonly integration: GitLabIntegration; private readonly logger: LoggerService; + private readonly events?: EventsService; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; private userTransformer: UserTransformer; @@ -77,6 +112,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; userTransformer?: UserTransformer; @@ -96,7 +132,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const integration = integrations.byHost(providerConfig.host); if (!providerConfig.orgEnabled) { - return; + throw new Error(`Org not enabled for ${providerConfig.id}.`); } if (!integration) { @@ -137,6 +173,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { config: GitlabProviderConfig; integration: GitLabIntegration; logger: LoggerService; + events?: EventsService; taskRunner: TaskRunner; userTransformer?: UserTransformer; groupEntitiesTransformer?: GroupEntitiesTransformer; @@ -148,6 +185,8 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { target: this.getProviderName(), }); this.scheduleFn = this.createScheduleFn(options.taskRunner); + this.events = options.events; + this.userTransformer = options.userTransformer ?? defaultUserTransformer; this.groupEntitiesTransformer = options.groupEntitiesTransformer ?? defaultGroupEntitiesTransformer; @@ -162,6 +201,117 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { async connect(connection: EntityProviderConnection): Promise { this.connection = connection; await this.scheduleFn(); + + // Specifies which topics will be listened to. + // The topics from the original GitLab events contain only the string 'gitlab'. These are caught by the GitlabEventRouter Module, who republishes them with a more specific topic 'gitlab.' + if (this.events) { + await this.events.subscribe({ + id: this.getProviderName(), + topics: [ + TOPIC_GROUP_CREATE, + TOPIC_GROUP_DESTROY, + TOPIC_GROUP_RENAME, + TOPIC_USER_CREATE, + TOPIC_USER_DESTROY, + TOPIC_USER_ADD_GROUP, + TOPIC_USER_REMOVE_GROUP, + ], + onEvent: async params => { + this.logger.info(`Received event from topic ${params.topic}`); + + const addEntitiesOperation = (entities: Entity[]) => ({ + removed: [], + added: entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), + })), + }); + + const removeEntitiesOperation = (entities: Entity[]) => ({ + added: [], + removed: entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), + })), + }); + + const replaceEntitiesOperation = (entities: Entity[]) => { + const entitiesToReplace = entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), + })); + + return { + removed: entitiesToReplace, + added: entitiesToReplace, + }; + }; + + // handle group change events + if ( + params.topic === TOPIC_GROUP_CREATE || + params.topic === TOPIC_GROUP_DESTROY + ) { + const payload: SystemHookGroupCreateOrDestroyEventSchema = + params.eventPayload as SystemHookGroupCreateOrDestroyEventSchema; + + const createDeltaOperation = + params.topic === TOPIC_GROUP_CREATE + ? addEntitiesOperation + : removeEntitiesOperation; + + await this.onGroupChange(payload, createDeltaOperation); + } + if (params.topic === TOPIC_GROUP_RENAME) { + const payload: SystemHookGroupRenameEventSchema = + params.eventPayload as SystemHookGroupRenameEventSchema; + + await this.onGroupEdit(payload, replaceEntitiesOperation); + } + + // handle user change events + if ( + params.topic === TOPIC_USER_CREATE || + params.topic === TOPIC_USER_DESTROY + ) { + const payload: SystemHookUserCreateOrDestroyEventSchema = + params.eventPayload as SystemHookUserCreateOrDestroyEventSchema; + + const createDeltaOperation = + params.topic === TOPIC_USER_CREATE + ? addEntitiesOperation + : removeEntitiesOperation; + + await this.onUserChange(payload, createDeltaOperation); + } + + // handle user membership changes + if ( + params.topic === TOPIC_USER_ADD_GROUP || + params.topic === TOPIC_USER_REMOVE_GROUP + ) { + const payload: SystemHookCreateOrDestroyMembershipEventsSchema = + params.eventPayload as SystemHookCreateOrDestroyMembershipEventsSchema; + + const createDeltaOperation = addEntitiesOperation; + + await this.onMembershipChange(payload, createDeltaOperation); + } + }, + }); + } } private createScheduleFn(taskRunner: TaskRunner): () => Promise { @@ -208,6 +358,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groups = paginated(options => client.listGroups(options), { page: 1, per_page: 100, + all_available: true, }); users = paginated(options => client.listUsers(options), { @@ -229,7 +380,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const idMappedUser: { [userId: number]: GitLabUser } = {}; - const res: Result = { + const userRes: UserResult = { scanned: 0, matches: [], }; @@ -240,46 +391,42 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }; for await (const user of users) { - if (!this.config.userPattern.test(user.email ?? user.username ?? '')) { - continue; - } + userRes.scanned++; - res.scanned++; - - if (user.state !== 'active') { + if (!this.shouldProcessUser(user)) { + logger.debug(`Skipped user: ${user.username}`); continue; } idMappedUser[user.id] = user; - res.matches.push(user); + userRes.matches.push(user); } for await (const group of groups) { - if (!this.config.groupPattern.test(group.full_path ?? '')) { - continue; - } - - if ( - this.config.group && - !group.full_path.startsWith(`${this.config.group}/`) - ) { - continue; - } - groupRes.scanned++; + + if (!this.shouldProcessGroup(group)) { + logger.debug(`Skipped group: ${group.full_path}`); + continue; + } + logger.debug(`Processed group: ${group.full_path}`); + groupRes.matches.push(group); let groupUsers: PagedResponse = { items: [] }; try { - groupUsers = await client.getGroupMembers(group.full_path, ['DIRECT']); + const relations = this.config.allowInherited + ? ['DIRECT', 'INHERITED'] + : ['DIRECT']; + groupUsers = await client.getGroupMembers(group.full_path, relations); } catch (e) { logger.error( `Failed fetching users for group '${group.full_path}': ${e}`, ); } - for (const groupUser of groupUsers.items) { const user = idMappedUser[groupUser.id]; + if (user) { user.groups = (user.groups ?? []).concat(group); } @@ -288,13 +435,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const groupsWithUsers = groupRes.matches.filter(group => { return ( - res.matches.filter(x => { + userRes.matches.filter(x => { return !!x.groups?.find(y => y.id === group.id); }).length > 0 ); }); - const userEntities = res.matches.map(p => + const userEntities = userRes.matches.map(p => this.userTransformer({ user: p, integrationConfig: this.integration.config, @@ -309,6 +456,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupNameTransformer: this.groupNameTransformer, }); + logger.info( + `Scanned ${userRes.scanned} users and processed ${userEntities.length} users`, + ); + logger.info( + `Scanned ${groupRes.scanned} groups and processed ${groupEntities.length} groups`, + ); + await this.connection.applyMutation({ type: 'full', entities: [...userEntities, ...groupEntities].map(entity => ({ @@ -321,6 +475,301 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { })), }); } + private async onGroupChange( + event: SystemHookGroupCreateOrDestroyEventSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const getClient = () => + new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + let group: GitLabGroup | undefined; + if (event.event_name === 'group_destroy') { + group = { + id: event.group_id, + full_path: event.full_path, + name: event.name, + description: '', + parent_id: 0, + }; + } else { + const client = getClient(); + group = await client.getGroupById(event.group_id); + } + + if (!this.shouldProcessGroup(group)) { + this.logger.debug(`Skipped group ${group.full_path}.`); + return; + } + + // create the group entity + const groupEntity = this.groupEntitiesTransformer({ + groups: [group], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + // we need to fetch the parent group's object because its representation might be changed by the groupTransformer + if (group.parent_id) { + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + const parentGroup = await client.getGroupById(group.parent_id); + + groupEntity[0].spec.parent = this.groupNameTransformer({ + group: parentGroup, + providerConfig: this.config, + }); + } + + this.logger.debug(`Applying mutation for group ${group.full_path}.`); + await this.connection.applyMutation({ + type: 'delta', + ...createDeltaOperation(groupEntity), + }); + } + + // the goal here is to trigger a mutation to remove the old entity and add the new one. + private async onGroupEdit( + event: SystemHookGroupRenameEventSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const groupToRemove: GitLabGroup = { + id: event.group_id, + full_path: event.old_full_path, + name: event.name, + description: '', + parent_id: 0, + }; + + if (!this.shouldProcessGroup(groupToRemove)) { + this.logger.debug(`Skipped group ${groupToRemove.full_path}.`); + return; + } + + const groupEntityToRemove = await this.groupEntitiesTransformer({ + groups: [groupToRemove], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + const groupToAdd = await client.getGroupById(event.group_id); + + if (!this.shouldProcessGroup(groupToAdd)) { + this.logger.debug(`Skipped group ${groupToAdd.full_path}.`); + return; + } + + const groupEntityToAdd = await this.groupEntitiesTransformer({ + groups: [groupToAdd], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + if (groupToAdd.parent_id) { + const parentGroup = await client.getGroupById(groupToAdd.parent_id); + + groupEntityToAdd[0].spec.parent = this.groupNameTransformer({ + group: parentGroup, + providerConfig: this.config, + }); + } + + const { added } = createDeltaOperation([...groupEntityToAdd]); + const { removed } = createDeltaOperation([...groupEntityToRemove]); + + this.logger.debug(`Applying mutation for group ${groupToAdd.full_path}.`); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onUserChange( + event: SystemHookUserCreateOrDestroyEventSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + let user: GitLabUser | undefined; + if (event.event_name === 'user_destroy') { + user = { + id: event.user_id, + username: event.username, + email: event.email, + name: event.name, + state: 'active', // in the delete case it doesn't really matter if the user is active or not + web_url: '', + avatar_url: '', + }; + } else { + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + user = await client.getUserById(event.user_id); + } + + if (!this.shouldProcessUser(user)) { + this.logger.debug(`Skipped user ${user.username}.`); + return; + } + + const userEntity = await this.userTransformer({ + user: user, + integrationConfig: this.integration.config, + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + const { added, removed } = createDeltaOperation([userEntity]); + + this.logger.debug(`Applying mutation for user ${user.username}.`); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + // the goal here is to reconstruct the group either from which the user was removed or to which the user was added. Specifically, we add/remove the new user to/from the spec.member property array of the group entity. The Processor should take care of updating the relations + private async onMembershipChange( + event: SystemHookCreateOrDestroyMembershipEventsSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + const groupToRebuild: GitLabGroup = await client.getGroupById( + event.group_id, + ); + + // If the group is outside the scope there is no point creating anything related to it. + if (!this.shouldProcessGroup(groupToRebuild)) { + this.logger.debug(`Skipped group ${groupToRebuild.full_path}.`); + return; + } + + const relations = this.config.allowInherited + ? ['DIRECT', 'INHERITED'] + : ['DIRECT']; + + const groupMembers = await client.getGroupMembers( + groupToRebuild.full_path, + relations, + ); + + // if (groupMembers.items.length !== 0) { + // groupMembers.items.forEach(element => { + // if ( + // event.event_name === 'user_remove_from_group' && + // element.username === event.user_username + // ) { + // return; + // } + + // usersToBeAdded.push(element.username); + // }); + // } + + // new members of the group + const usersToBeAdded = + event.event_name === 'user_remove_from_group' + ? groupMembers.items.filter(e => e.username !== event.user_username) + : groupMembers.items; + + const groupEntityToModify = await this.groupEntitiesTransformer({ + groups: [groupToRebuild], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + // we need to fetch the parent group's object because its representation might be changed by the groupTransformer + if (groupToRebuild.parent_id) { + const parentGroup = await client.getGroupById(groupToRebuild.parent_id); + + groupEntityToModify[0].spec.parent = this.groupNameTransformer({ + group: parentGroup, + providerConfig: this.config, + }); + } + + if (usersToBeAdded.length !== 0) { + groupEntityToModify[0].spec.members = usersToBeAdded.map(e => e.username); + } + + const { added, removed } = createDeltaOperation([...groupEntityToModify]); + + this.logger.debug( + `Applying mutation for group ${groupToRebuild.full_path}.`, + ); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private shouldProcessGroup(group: GitLabGroup): boolean { + return ( + this.config.groupPattern.test(group.full_path) && + (!this.config.group || + group.full_path.startsWith(`${this.config.group}/`)) + ); + } + + private shouldProcessUser(user: GitLabUser): boolean { + return ( + this.config.userPattern.test(user.email ?? user.username ?? '') && + user.state === 'active' + ); + } private withLocations(host: string, baseUrl: string, entity: Entity): Entity { const location = diff --git a/yarn.lock b/yarn.lock index ad5d48bd8f..ce5c906b86 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5569,7 +5569,24 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab": +"@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend-test-utils": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + luxon: ^3.0.0 + languageName: unknown + linkType: soft + +"@backstage/plugin-catalog-backend-module-gitlab@workspace:^, @backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab" dependencies: @@ -5581,7 +5598,11 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend-test-utils": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + "@gitbeaker/rest": ^39.25.0 "@types/lodash": ^4.14.151 "@types/uuid": ^9.0.0 lodash: ^4.17.21