From a70377d3e1902480cbae14b2c7a861514f0a61ff Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Sat, 16 Mar 2024 12:15:00 +0100 Subject: [PATCH 01/15] feat: add events support for gitlab catalog entity provider Signed-off-by: ElaineDeMattosSilvaB --- .changeset/eleven-pandas-divide.md | 5 + .changeset/shy-students-clap.md | 5 + .../.eslintrc.js | 1 + .../README.md | 8 + .../api-report.md | 11 + .../catalog-info.yaml | 10 + .../package.json | 47 + ...leGitlabOrgDiscoveryEntityProvider.test.ts | 109 ++ ...gModuleGitlabOrgDiscoveryEntityProvider.ts | 55 + .../src/index.ts | 22 + .../CHANGELOG.md | 12 +- .../api-report.md | 5 +- .../package.json | 56 +- .../src/__testUtils__/handlers.ts | 521 +++++ .../src/__testUtils__/mocks.ts | 1721 +++++++++++++++++ .../src/lib/client.test.ts | 905 +++------ .../src/lib/client.ts | 77 +- .../src/lib/defaultTransformers.ts | 4 - .../src/lib/index.ts | 6 +- .../src/lib/types.ts | 79 +- ...oduleGitlabDiscoveryEntityProvider.test.ts | 28 +- ...alogModuleGitlabDiscoveryEntityProvider.ts | 12 +- .../GitlabDiscoveryEntityProvider.test.ts | 819 +++----- .../GitlabDiscoveryEntityProvider.ts | 331 +++- .../GitlabOrgDiscoveryEntityProvider.test.ts | 1254 +++++------- .../GitlabOrgDiscoveryEntityProvider.ts | 515 ++++- yarn.lock | 23 +- 27 files changed, 4597 insertions(+), 2044 deletions(-) create mode 100644 .changeset/eleven-pandas-divide.md create mode 100644 .changeset/shy-students-clap.md create mode 100644 plugins/catalog-backend-module-gitlab-org/.eslintrc.js create mode 100644 plugins/catalog-backend-module-gitlab-org/README.md create mode 100644 plugins/catalog-backend-module-gitlab-org/api-report.md create mode 100644 plugins/catalog-backend-module-gitlab-org/catalog-info.yaml create mode 100644 plugins/catalog-backend-module-gitlab-org/package.json create mode 100644 plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts create mode 100644 plugins/catalog-backend-module-gitlab-org/src/index.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts diff --git a/.changeset/eleven-pandas-divide.md b/.changeset/eleven-pandas-divide.md new file mode 100644 index 0000000000..912d34570a --- /dev/null +++ b/.changeset/eleven-pandas-divide.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Added events support for `GitlabDiscoveryEntityProvider` and `GitlabOrgDiscoveryEntityProvider`. diff --git a/.changeset/shy-students-clap.md b/.changeset/shy-students-clap.md new file mode 100644 index 0000000000..0e6244f21d --- /dev/null +++ b/.changeset/shy-students-clap.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab-org': patch +--- + +Added a new `catalog-backend-module-gitlab-org` module which adds the `GitlabOrgDiscoveryEntityProvider` to the catalog's providers using the new backend system. diff --git a/plugins/catalog-backend-module-gitlab-org/.eslintrc.js b/plugins/catalog-backend-module-gitlab-org/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-gitlab-org/README.md b/plugins/catalog-backend-module-gitlab-org/README.md new file mode 100644 index 0000000000..0b86b8d815 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/README.md @@ -0,0 +1,8 @@ +# Catalog Backend Module for GitLab Organizational Data + +This is an extension module to the plugin-catalog-backend plugin that provides group and user entities by scrapping or receiving events from a GitLab instance. + +## Getting started + +See [Backstage documentation](https://backstage.io/docs/integrations/gitlab/org) for details on how to install +and configure the plugin. diff --git a/plugins/catalog-backend-module-gitlab-org/api-report.md b/plugins/catalog-backend-module-gitlab-org/api-report.md new file mode 100644 index 0000000000..2a36f49206 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/api-report.md @@ -0,0 +1,11 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-gitlab-org" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; + +// @public +const catalogModuleGitlabOrgDiscoveryEntityProvider: () => BackendFeature; +export default catalogModuleGitlabOrgDiscoveryEntityProvider; +``` diff --git a/plugins/catalog-backend-module-gitlab-org/catalog-info.yaml b/plugins/catalog-backend-module-gitlab-org/catalog-info.yaml new file mode 100644 index 0000000000..1846e3a9df --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-catalog-backend-module-gitlab-org + title: '@backstage/plugin-catalog-backend-module-gitlab-org' + description: The gitlab-org backend module for the catalog plugin. +spec: + lifecycle: experimental + type: backstage-backend-plugin-module + owner: maintainers diff --git a/plugins/catalog-backend-module-gitlab-org/package.json b/plugins/catalog-backend-module-gitlab-org/package.json new file mode 100644 index 0000000000..7d92fda325 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/package.json @@ -0,0 +1,47 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-gitlab-org", + "version": "0.0.0", + "description": "The gitlab-org backend module for the catalog plugin.", + "backstage": { + "role": "backend-plugin-module" + }, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-gitlab-org" + }, + "license": "Apache-2.0", + "main": "src/index.ts", + "types": "src/index.ts", + "files": [ + "dist" + ], + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^" + }, + "devDependencies": { + "@backstage/backend-tasks": "workspace:^", + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@backstage/plugin-events-backend-test-utils": "workspace:^", + "luxon": "^3.0.0" + } +} diff --git a/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts new file mode 100644 index 0000000000..568c91c8e2 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createServiceFactory } from '@backstage/backend-plugin-api'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; +import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { TestEventsService } from '@backstage/plugin-events-backend-test-utils'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; +import { Duration } from 'luxon'; +import { catalogModuleGitlabOrgDiscoveryEntityProvider } from './catalogModuleGitlabOrgDiscoveryEntityProvider'; + +describe('catalogModuleGitlabOrgDiscoveryEntityProvider', () => { + it('should register provider at the catalog extension point', async () => { + const events = new TestEventsService(); + const eventsServiceFactory = createServiceFactory({ + service: eventsServiceRef, + deps: {}, + async factory({}) { + return events; + }, + }); + let addedProviders: Array | undefined; + let usedSchedule: TaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const connection = jest.fn() as unknown as EntityProviderConnection; + const runner = jest.fn(); + const scheduler = mockServices.scheduler.mock({ + createScheduledTaskRunner(schedule) { + usedSchedule = schedule; + return { run: runner }; + }, + }); + + const config = { + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + orgEnabled: true, + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }; + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + features: [ + eventsServiceFactory(), + catalogModuleGitlabOrgDiscoveryEntityProvider(), + mockServices.rootConfig.factory({ data: config }), + mockServices.logger.factory(), + scheduler.factory, + ], + }); + + expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); + expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); + expect(addedProviders?.length).toEqual(1); + expect(runner).not.toHaveBeenCalled(); + + const provider = addedProviders!.pop()!; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + await provider.connect(connection); + expect(events.subscribed).toHaveLength(1); + expect(events.subscribed[0].id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + expect(runner).toHaveBeenCalledTimes(1); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts new file mode 100644 index 0000000000..ffb2cfb629 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; + +/** + * Registers the GitlabOrgDiscoveryEntityProvider with the catalog processing extension point. + * + * @public + */ +export const catalogModuleGitlabOrgDiscoveryEntityProvider = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'gitlabOrgDiscoveryEntityProvider', + register(env) { + env.registerInit({ + deps: { + config: coreServices.rootConfig, + catalog: catalogProcessingExtensionPoint, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + events: eventsServiceRef, + }, + async init({ config, catalog, logger, scheduler, events }) { + const gitlabOrgDiscoveryEntityProvider = + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger: loggerToWinstonLogger(logger), + events, + scheduler, + }); + catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider); + }, + }); + }, + }); diff --git a/plugins/catalog-backend-module-gitlab-org/src/index.ts b/plugins/catalog-backend-module-gitlab-org/src/index.ts new file mode 100644 index 0000000000..bbd5f1f4c3 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab-org/src/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The gitlab-org backend module for the catalog plugin. + * + * @packageDocumentation + */ +export { catalogModuleGitlabOrgDiscoveryEntityProvider as default } from './catalogModuleGitlabOrgDiscoveryEntityProvider'; diff --git a/plugins/catalog-backend-module-gitlab/CHANGELOG.md b/plugins/catalog-backend-module-gitlab/CHANGELOG.md index 4e8493d5d6..92f0ea7f21 100644 --- a/plugins/catalog-backend-module-gitlab/CHANGELOG.md +++ b/plugins/catalog-backend-module-gitlab/CHANGELOG.md @@ -385,8 +385,8 @@ ref: - https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api - https://docs.gitlab.com/ee/api/members.html#limitations + + - 890e3b5ad4: Make sure to include the error message when ingestion fails - 0b55f773a7: Removed some unused dependencies @@ -417,8 +417,8 @@ ref: - https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api - https://docs.gitlab.com/ee/api/members.html#limitations + + - 0b55f773a7: Removed some unused dependencies - Updated dependencies @@ -1211,7 +1211,7 @@ - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. Please find how to configure the schedule at the config at - https://backstage.io/docs/integrations/gitlab/discovery + - 4c9f7847e4: Updated dependency `msw` to `^0.48.0` while moving it to be a dev dependency. - Updated dependencies @@ -1250,7 +1250,7 @@ - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. Please find how to configure the schedule at the config at - https://backstage.io/docs/integrations/gitlab/discovery + - Updated dependencies - @backstage/backend-common@0.16.0-next.0 diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index cd9ade4bfb..809ac168f2 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -8,6 +8,7 @@ import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { EventsService } from '@backstage/plugin-events-node'; import { GitLabIntegrationConfig } from '@backstage/integration'; import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/plugin-catalog-node'; @@ -26,6 +27,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; }, @@ -63,7 +65,6 @@ export type GitLabGroup = { name: string; full_path: string; description?: string; - visibility?: string; parent_id?: number; }; @@ -81,6 +82,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; userTransformer?: UserTransformer; @@ -103,6 +105,7 @@ export type GitlabProviderConfig = { projectPattern: RegExp; userPattern: RegExp; groupPattern: RegExp; + allowInherited?: boolean; orgEnabled?: boolean; schedule?: TaskScheduleDefinition; skipForkedRepos?: boolean; diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 75bccd4faa..54ecf67503 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -1,5 +1,21 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ { "name": "@backstage/plugin-catalog-backend-module-gitlab", + "version": "0.3.10-next.0", "description": "A Backstage catalog backend module that helps integrate towards GitLab", "version": "0.3.15", "main": "src/index.ts", @@ -8,11 +24,23 @@ "publishConfig": { "access": "public" }, + "keywords": [ + "backstage" + ], + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-gitlab" + }, + "license": "Apache-2.0", "exports": { ".": "./src/index.ts", "./alpha": "./src/alpha.ts", "./package.json": "./package.json" }, + "main": "src/index.ts", + "types": "src/index.ts", "typesVersions": { "*": { "alpha": [ @@ -23,26 +51,18 @@ ] } }, - "backstage": { - "role": "backend-plugin-module" - }, - "homepage": "https://backstage.io", - "repository": { - "type": "git", - "url": "https://github.com/backstage/backstage", - "directory": "plugins/catalog-backend-module-gitlab" - }, - "keywords": [ - "backstage" + "files": [ + "config.d.ts", + "dist" ], "scripts": { - "start": "backstage-cli package start", "build": "backstage-cli package build", + "clean": "backstage-cli package clean", "lint": "backstage-cli package lint", - "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", - "clean": "backstage-cli package clean" + "start": "backstage-cli package start", + "test": "backstage-cli package test" }, "dependencies": { "@backstage/backend-common": "workspace:^", @@ -51,7 +71,10 @@ "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/integration": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", + "@gitbeaker/rest": "^39.25.0", "lodash": "^4.17.21", "node-fetch": "^2.6.7", "uuid": "^9.0.0" @@ -59,14 +82,11 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", + "@backstage/plugin-events-backend-test-utils": "workspace:^", "@types/lodash": "^4.14.151", "@types/uuid": "^9.0.0", "luxon": "^3.0.0", "msw": "^1.0.0" }, - "files": [ - "config.d.ts", - "dist" - ], "configSchema": "config.d.ts" } diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts new file mode 100644 index 0000000000..2f464dc1fe --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts @@ -0,0 +1,521 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { graphql, rest } from 'msw'; +import { + all_groups_response, + all_projects_response, + all_saas_users_response, + all_users_response, + apiBaseUrl, + apiBaseUrlSaas, + graphQlBaseUrl, + paged_endpoint, + saasGraphQlBaseUrl, + some_endpoint, + unhealthy_endpoint, + userID, +} from './mocks'; + +const httpHandlers = [ + /** + * Project REST endpoint mocks + */ + + // fetch all projects in an instance + rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => { + return res(ctx.set('x-next-page', ''), ctx.json(all_projects_response)); + }), + + rest.get(`${apiBaseUrl}/projects/42`, (_, res, ctx) => { + return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); + }), + + // testing non existing file + rest.get( + `${apiBaseUrl}/projects/test-group%2Ftest-repo1/repository/files/catalog-info.yaml`, + (_, res, ctx) => { + return res(ctx.status(400), ctx.json({ error: 'Not found' })); + }, + ), + + /** + * Group REST endpoint mocks + */ + + rest.get(`${apiBaseUrl}/groups`, (_req, res, ctx) => { + return res(ctx.set('x-next-page', ''), ctx.json(all_groups_response)); + }), + + rest.get(`${apiBaseUrl}/groups/42`, (_, res, ctx) => { + return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); + }), + + rest.get(`${apiBaseUrlSaas}/groups/group1/members/all`, (_req, res, ctx) => { + return res(ctx.json(all_saas_users_response)); + }), + + /** + * Users REST endpoint mocks + */ + + rest.get(`${apiBaseUrl}/users`, (_req, res, ctx) => { + return res(ctx.set('x-next-page', ''), ctx.json(all_users_response)); + }), + + rest.get(`${apiBaseUrl}/users/${userID}`, (_, res, ctx) => { + return res(ctx.json(all_users_response.find(user => user.id === userID))); + }), + + rest.get(`${apiBaseUrl}/users/42`, (_, res, ctx) => { + return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })); + }), + + /** + * others + */ + + // mock a 4 page response + rest.get(`${apiBaseUrl}${paged_endpoint}`, (req, res, ctx) => { + const page = req.url.searchParams.get('page'); + const currentPage = page ? Number(page) : 1; + const fakePageCount = 4; + + return res( + // set next page number header if page requested is less than count + ctx.set( + 'x-next-page', + currentPage < fakePageCount ? String(currentPage + 1) : '', + ), + ctx.json([{ someContentOfPage: currentPage }]), + ); + }), + + rest.get(`${apiBaseUrl}${unhealthy_endpoint}`, (_, res, ctx) => { + return res(ctx.status(400), ctx.json({ error: 'some error' })); + }), + + rest.get(`${apiBaseUrl}${some_endpoint}`, (req, res, ctx) => { + return res(ctx.json([{ endpoint: req.url.toString() }])); + }), +]; + +// dynamic handlers + +const httpGroupFindByIdDynamic = all_groups_response.map(group => { + return rest.get(`${apiBaseUrl}/groups/${group.id}`, (_, res, ctx) => { + return res(ctx.json(all_groups_response.find(g => g.id === group.id))); + }); +}); + +const httpGroupListDescendantProjectsById = all_groups_response.map(group => { + return rest.get( + `${apiBaseUrl}/groups/${group.id}/projects`, + (_, res, ctx) => { + const projectsInGroup = all_projects_response.filter(p => + p.path_with_namespace?.includes(group.name), + ); + + return res(ctx.json(projectsInGroup)); + }, + ); +}); + +const httpGroupListDescendantProjectsByName = all_groups_response.map(group => { + return rest.get( + `${apiBaseUrl}/groups/${group.name}/projects`, + (_, res, ctx) => { + const projectsInGroup = all_projects_response.filter(p => + p.path_with_namespace?.includes(group.name), + ); + + return res(ctx.json(projectsInGroup)); + }, + ); +}); +const httpGroupFindByNameDynamic = all_groups_response.map(group => { + return rest.get(`${apiBaseUrl}/groups/${group.name}`, (_, res, ctx) => { + return res(ctx.json(all_groups_response.find(g => g.name === group.name))); + }); +}); +const httpProjectFindByIdDynamic = all_projects_response.map(project => { + return rest.get(`${apiBaseUrl}/projects/${project.id}`, (_, res, ctx) => { + return res(ctx.json(all_projects_response.find(p => p.id === project.id))); + }); +}); +const httpProjectCatalogDynamic = all_projects_response.map(project => { + const path: string = project.path_with_namespace + ? project.path_with_namespace!.replace(/\//g, '%2F') + : `${project.path_with_namespace}%2F${project.name}`; + + return rest.head( + `${apiBaseUrl}/projects/${path}/repository/files/catalog-info.yaml`, + (req, res, ctx) => { + const branch = req.url.searchParams.get('ref'); + if (branch === project.default_branch) { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ); +}); + +/** + * GraphQL endpoint mocks + */ + +const graphqlHandlers = [ + graphql + .link(graphQlBaseUrl) + .query('getGroupMembers', async (req, res, ctx) => { + const { group, relations } = req.variables; + + if (group === 'group1' && relations.includes('DIRECT')) { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + if (group === 'saas-multi-user-group') { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + publicEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + + if (group === 'non-existing-group' || group === '') { + return res( + ctx.data({ + group: { + groupMembers: { + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + + if (group === 'multi-page' && relations.includes('DIRECT')) { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: req.variables.endCursor + ? [{ user: { id: 'gid://gitlab/User/2' } }] + : [{ user: { id: 'gid://gitlab/User/1' } }], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ); + } + + if (group === 'multi-page-saas') { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: req.variables.endCursor + ? [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ] + : [ + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + publicEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ); + } + + if (group === 'error-group') { + return res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ); + } + + return res(ctx.status(200)); + }), + + graphql + .link(graphQlBaseUrl) + .query('listDescendantGroups', async (req, res, ctx) => { + const { group } = req.variables; + + if (group === 'error-group') { + return res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ); + } + + if (group === 'group-with-parent') { + return res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/1', + name: 'group-with-parent', + description: 'description1', + fullPath: 'path/group-with-parent', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + if (group === 'root') { + return res( + ctx.data({ + group: { + descendantGroups: { + nodes: req.variables.endCursor + ? [ + { + id: 'gid://gitlab/Group/1', + name: 'group1', + description: 'description1', + fullPath: 'path/group1', + parent: { + id: '123', + }, + }, + ] + : [ + { + id: 'gid://gitlab/Group/2', + name: 'group2', + description: 'description2', + fullPath: 'path/group2', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ); + } + + if (group === 'non-existing-group') { + return res( + ctx.data({ + group: {}, + }), + ); + } + return res(ctx.status(200)); + }), + + graphql + .link(saasGraphQlBaseUrl) + .query('getGroupMembers', async (req, res, ctx) => { + const { group } = req.variables; + + if (group === 'saas-multi-user-group') { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + publicEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } + + if (group === '') { + return res(ctx.data({})); + } + + if (group === 'error-group') { + return res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ); + } + + return res(ctx.status(200)); + }), + + graphql + .link(saasGraphQlBaseUrl) + .query('listDescendantGroups', async (_, res, ctx) => { + return res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/456', + name: 'group1', + description: 'Group1', + fullPath: 'group1/group1', + parent: { + id: 'gid://gitlab/Group/123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + }), +]; + +export const handlers = [ + ...httpHandlers, + ...httpProjectFindByIdDynamic, + ...httpProjectCatalogDynamic, + ...httpGroupFindByIdDynamic, + ...httpGroupFindByNameDynamic, + ...httpGroupListDescendantProjectsById, + ...httpGroupListDescendantProjectsByName, + ...graphqlHandlers, +]; diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts new file mode 100644 index 0000000000..9e0f55d05d --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts @@ -0,0 +1,1721 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { locationSpecToMetadataName } from '@backstage/plugin-catalog-node'; +import { EventParams } from '@backstage/plugin-events-node'; +import { GitLabGroup, GitLabProject, GitLabUser } from '../lib/types'; + +type MockObject = Record; + +/** + * General + */ +export const apiBaseUrl: string = 'https://example.com/api/v4'; +export const apiBaseUrlSaas: string = 'https://gitlab.com/api/v4'; +export const graphQlBaseUrl = 'https://example.com/api/graphql'; +export const saasGraphQlBaseUrl = 'https://gitlab.com/api/graphql'; +export const groupName: string = 'group1'; +export const groupID: number = 1; +export const userID: number = 1; +export const projectID: number = 1; + +/** + * Endpoints + */ + +export const paged_endpoint: string = `/paged-endpoint`; +export const some_endpoint: string = `/some-endpoint`; +export const unhealthy_endpoint = `/unhealthy-endpoint`; + +/** + * Integration Configurations + */ + +export const config_self_managed: MockObject = { + host: 'example.com', + token: 'test-token', + apiBaseUrl: 'https://example.com/api/v4', + baseUrl: 'https://example.com', +}; + +export const config_saas: MockObject = { + host: 'gitlab.com', + token: 'test-token', + apiBaseUrl: 'https://gitlab.com/api/v4', + baseUrl: 'https://gitlab.com', +}; + +export const config_no_org_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_disabled_org_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + skipForkedRepos: false, + orgEnabled: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_saas_no_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + skipForkedRepos: false, + orgEnabled: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_non_gitlab_host: MockObject = { + integrations: { + github: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_single_integration_branch: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + branch: 'main', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_single_integration_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'nonMatchingGroup', + branch: 'main', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_fallbackBranch_branch: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + fallbackBranch: 'staging', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_single_integration_skip_forks: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + skipForkedRepos: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_no_schedule: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + orgEnabled: true, + skipForkedRepos: true, + }, + }, + }, + }, +}; + +export const config_single_integration_project_pattern: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + projectPattern: 'test-repo', + skipForkedRepos: false, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_no_schedule_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + projectPattern: 'test-repo', + skipForkedRepos: true, + }, + }, + }, + }, +}; + +export const config_unmatched_project_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'my-other-group', + projectPattern: 'my-other-project', + skipForkedRepos: true, + }, + }, + }, + }, +}; + +export const config_double_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + }, + 'second-test': { + host: 'example.com', + group: 'second-group', + }, + }, + }, + }, +}; + +export const config_org_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_userPattern_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + userPattern: '^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+.[a-zA-Z]{2,}$', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_saas: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + group: 'group1', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_no_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_saas_no_group: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + groupPattern: '^group.*', + orgEnabled: true, + }, + }, + }, + }, +}; + +export const config_org_integration_saas_sched: MockObject = { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + group: 'group1', + groupPattern: '^group.*', + orgEnabled: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + +export const config_org_double_integration: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + orgEnabled: true, + }, + 'second-test': { + host: 'example.com', + group: 'second-group', + orgEnabled: true, + }, + }, + }, + }, +}; + +/** + * GitLab API responses + */ + +export const all_projects_response: GitLabProject[] = [ + { + id: 1, + description: 'Project One Description', + name: 'test-repo1', + default_branch: 'main', + path: 'test-repo1', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo1', + path_with_namespace: 'group1/test-repo1', + }, + { + id: 2, + description: 'Project Two Description', + name: 'test-repo2', + default_branch: 'prd', + path: 'test-repo2', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo2', + path_with_namespace: 'group1/test-repo2', + }, + // unmatched project + { + id: 3, + description: 'Project Three Description', + name: 'repo3', + default_branch: 'main', + path: 'repo3', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/repo3', + path_with_namespace: 'group1/repo3', + }, + // forked project + { + id: 4, + description: 'Project Four Description', + name: 'test-repo4-forked', + default_branch: 'main', + path: 'test-repo4-forked', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo4-forked', + path_with_namespace: 'group1/test-repo4-forked', + forked_from_project: { id: 13083 }, + }, + // fallBack branch + { + id: 5, + description: 'Project Five Description', + name: 'test-repo5-staging', + default_branch: 'staging', + path: 'test-repo5-staging', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo5-staging', + path_with_namespace: 'group1/test-repo5-staging', + }, + // diffrent group + { + id: 6, + description: 'Project Six Description', + name: 'test-repo6', + default_branch: 'main', + path: 'test-repo6', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo6', + path_with_namespace: 'awesome-group/test-repo6', + }, +]; + +export const all_users_response: GitLabUser[] = [ + { + id: 1, + username: 'JohnDoe', + name: 'John Doe', + state: 'active', + email: 'john.doe@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/john_doe', + }, + { + id: 2, + username: 'JaneDoe', + name: 'Jane Doe', + state: 'active', + email: 'jane.doe@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/jane_doe', + }, + { + id: 3, + username: 'MarySmith', + name: 'Mary Smith', + state: 'active', + email: 'mary.smith@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/mary_smith', + }, + // inactive user + { + id: 4, + username: 'LuigiMario', + name: 'Luigi Mario', + state: 'inactive', + email: 'luigi.mario@company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/luigi_mario', + }, + // malfomed email address + { + id: 5, + username: 'MarioMario', + name: 'Mario Mario', + state: 'active', + email: 'mario.mario-company.com', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/mario_mario', + }, +]; + +export const all_saas_users_response: MockObject[] = [ + { + access_level: 30, + created_at: '2023-07-17T08:58:34.984Z', + expires_at: null, + id: 12, + username: 'testuser1', + name: 'Test User 1', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.com/testuser1', + email: 'testuser1@example.com', + group_saml_identity: { + provider: 'group_saml', + extern_uid: '51', + saml_provider_id: 1, + }, + is_using_seat: true, + membership_state: 'active', + }, + { + access_level: 30, + created_at: '2023-07-19T08:58:34.984Z', + expires_at: null, + id: 34, + username: 'testuser2', + name: 'Test User 2', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.com/testuser2', + email: 'testuser2@example.com', + group_saml_identity: { + provider: 'group_saml', + extern_uid: '52', + saml_provider_id: 1, + }, + is_using_seat: true, + membership_state: 'active', + }, + { + access_level: 50, + created_at: '2023-07-15T08:58:34.984Z', + expires_at: '2023-10-26', + id: 54, + username: 'group_100_bot_23dc8057bef66e05181f39be4652577c', + name: 'Token Bot', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: + 'https://gitlab.com/group_100_bot_23dc8057bef66e05181f39be4652577c', + group_saml_identity: null, + is_using_seat: false, + membership_state: 'active', + }, +]; + +export const all_groups_response: GitLabGroup[] = [ + { + id: 1, + name: 'group1', + description: '', + full_path: 'group1', + }, + { + id: 2, + name: 'group2', + description: '', + full_path: 'group2', + }, + { + id: 3, + name: 'group3', + description: '', + full_path: 'group3', + }, + { + id: 4, + name: 'group1', + description: '', + full_path: 'group-new/group1', + }, + { + id: 5, + name: 'nonMatchingGroup', + description: '', + full_path: 'parent1/nonMatchingGroup', + }, +]; + +export const group_with_parent: MockObject[] = [ + { + id: 1, + name: 'group-with-parent', + description: 'description1', + full_path: 'path/group-with-parent', + parent_id: 123, + }, +]; + +export const expectedSaasMember: MockObject[] = [ + { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }, + { + id: 2, + username: 'user2', + email: 'user2@example.com', + name: 'user2', + state: 'active', + web_url: 'user2.com', + avatar_url: 'user2', + }, +]; + +export const expectedSaasGroup: MockObject[] = [ + { + id: 1, + name: 'group1', + description: 'description1', + full_path: 'path/group1', + parent_id: 123, + }, + { + id: 2, + name: 'group2', + description: 'description2', + full_path: 'path/group2', + parent_id: 123, + }, +]; + +/** + * GitLab Events + */ + +const added_commits: MockObject[] = [ + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: ['catalog-info.yaml'], + modified: [], + removed: [], + }, + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: ['cool-folder-1/cool-folder-2/catalog-info.yaml'], + modified: [], + removed: [], + }, +]; + +const removed_commits: MockObject[] = [ + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: [], + modified: [], + removed: ['catalog-info.yaml'], + }, +]; + +const modified_commits: MockObject[] = [ + { + id: 'ce53673ebe13a961a6b937411019e7c1db79741f', + message: 'test', + title: 'test', + timestamp: '2024-01-24T14:16:55+00:00', + url: 'https://example.com/group1/test-repo1/-/commit/ce53673ebe13a961a6b937411019e7c1db79741f', + author: { + name: 'Tom Sawyer', + email: 'tom.sawyer@email.com', + }, + added: [], + modified: ['catalog-info.yaml'], + removed: [], + }, +]; + +export const group_destroy_event: EventParams = { + topic: 'gitlab.group_destroy', + eventPayload: { + event_name: 'group_destroy', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'group1', + path: 'group1', + full_path: 'group1', + group_id: 123, + }, +}; + +export const group_destroy_event_unmatched: EventParams = { + topic: 'gitlab.group_destroy', + eventPayload: { + event_name: 'group_destroy', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'my-awesome-group', + path: 'my-awesome-group', + full_path: 'parent/my-awesome-group', + group_id: 123, + }, +}; + +export const group_rename_event: EventParams = { + topic: 'gitlab.group_rename', + eventPayload: { + event_name: 'group_rename', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'group1', // this is the displayname + path: 'group-new', + full_path: 'group-new/group1', + old_path: 'group-old', + old_full_path: 'group-old/group1', + group_id: 4, + }, +}; + +export const group_create_event: EventParams = { + topic: 'gitlab.group_create', + eventPayload: { + event_name: 'group_create', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'group1', + path: 'group1', + full_path: 'group1', + group_id: 1, + }, +}; +export const group_create_event_unmatched: EventParams = { + topic: 'gitlab.group_create', + eventPayload: { + event_name: 'group_create', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + name: 'nonMatchingGroup', + path: 'nonMatchingGroup', + full_path: 'parent1/nonMatchingGroup', + group_id: 5, + }, +}; + +export const user_create_event: EventParams = { + topic: 'gitlab.user_create', + eventPayload: { + event_name: 'user_create', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + email: 'john.doe@company.com', + name: 'John Doe', + username: 'johndoe', + user_id: 1, + }, +}; +export const user_destroy_event: EventParams = { + topic: 'gitlab.user_destroy', + eventPayload: { + event_name: 'user_destroy', + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + email: 'john.doe@company.com', + name: 'John Doe', + username: 'johndoe', + user_id: 1, + }, +}; + +export const user_add_to_group_event: EventParams = { + topic: 'gitlab.user_add_to_group', + eventPayload: { + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + group_name: 'group1', + group_path: 'my-groups/group1', + group_id: 1, + user_username: 'JohnDoe', + user_name: 'John Doe', + user_email: 'john.doe@company.com', + user_id: 1, + group_access: 'Owner', + expires_at: null, + group_plan: null, + event_name: 'user_add_to_group', + }, +}; + +export const user_add_to_group_event_mismatched: EventParams = { + topic: 'gitlab.user_add_to_group', + eventPayload: { + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + group_name: 'nonMatchingGroup', + group_path: 'parent1/nonMatchingGroup', + group_id: 5, + user_username: 'JohnDoe', + user_name: 'John Doe', + user_email: 'john.doe@company.com', + user_id: 1, + group_access: 'Owner', + expires_at: null, + group_plan: null, + event_name: 'user_add_to_group', + }, +}; + +export const user_remove_from_group_event: EventParams = { + topic: 'gitlab.user_remove_from_group', + eventPayload: { + created_at: '2024-02-02T10:53:09Z', + updated_at: '2024-02-02T10:53:09Z', + group_name: 'group1', + group_path: 'my-groups/group1', + group_id: 1, + user_username: 'user1', + user_name: 'John Doe', + user_email: 'john.doe@company.com', + user_id: 1, + group_access: 'Owner', + expires_at: null, + group_plan: null, + event_name: 'user_remove_from_group', + }, +}; + +export const push_add_event: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 1, + project: { + name: 'test-repo', + path_with_namespace: 'group1/test-repo1', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo1', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo1', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: added_commits, + total_commits_count: 2, + repository: { + name: 'test-repo1', + url: 'https://gitlab.com/group1/test-repo1', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo1', + git_http_url: 'https://gitlab.com/group1/test-repo1.git', + git_ssh_url: 'git@gitlab.com:group1/test-repo1.git', + visibility_level: 20, + }, + }, +}; + +export const push_add_event_unmatched_group: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 6, + project: { + name: 'test-repo6', + path_with_namespace: 'awesome-group/test-repo6', + description: 'My Cool Project', + web_url: 'https://example.com/awesome-group/test-repo6', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/awesome-group/test-repo6', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: added_commits, + total_commits_count: 2, + repository: { + name: 'test-repo6', + url: 'https://gitlab.com/awesome-group/test-repo6', + description: 'My Cool Project', + homepage: 'https://gitlab.com/awesome-group/test-repo6', + git_http_url: 'https://gitlab.com/awesome-group/test-repo6.git', + git_ssh_url: 'git@gitlab.com:awesome-group/test-repo6.git', + visibility_level: 20, + }, + }, +}; +export const push_add_event_forked: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 4, + project: { + name: 'test-repo4-forked', + path_with_namespace: 'group1/test-repo4-forked', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo4-forked', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo4-forked', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: added_commits, + total_commits_count: 2, + repository: { + name: 'test-repo4-forked', + url: 'https://gitlab.com/group1/test-repo4-forked', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo4-forked', + git_http_url: 'https://gitlab.com/group1/test-repo4-forked.git', + git_ssh_url: 'git@gitlab.com:group1/test-repo4-forked.git', + visibility_level: 20, + }, + }, +}; + +export const push_remove_event: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 1, + project: { + name: 'test-repo', + path_with_namespace: 'group1/test-repo1', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo1', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo1', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: removed_commits, + total_commits_count: 2, + repository: { + name: 'test-repo', + url: 'https://gitlab.com/group1/test-repo1', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo1', + git_http_url: 'https://gitlab.com/group1/test-repo1.git', + git_ssh_url: 'git@gitlab.com:bgroup1/test-repo1.git', + visibility_level: 20, + }, + }, +}; + +export const push_modif_event: EventParams = { + topic: 'gitlab.push', + metadata: { + 'x-gitlab-event': 'Push Hook', + }, + eventPayload: { + object_kind: 'push', + event_name: 'push', + before: 'a1a1472b4a1b51d521d75a95cethisisatest00', + after: '616thisisatestc424d5031540dee772a845bcf9', + ref: 'refs/heads/main', + ref_protected: true, + checkout_sha: '616c427c283fb1b834d5thisiatest72a845bcf9', + user_id: 11013327, + user_name: 'Tom Sawyer', + user_username: 'tom.sawyer', + user_email: 'tom.sawyer@email.com', + user_avatar: 'https://secure.gravatar.com/avatar/testtest=42&d=identicon', + project_id: 1, + project: { + name: 'test-repo', + path_with_namespace: 'group1/test-repo1', + description: 'My Cool Project', + web_url: 'https://example.com/group1/test-repo1', + avatar_url: null, + namespace: 'group1', + visibility_level: 20, + default_branch: 'main', + url: 'https://example.com/group1/test-repo1', + git_ssh_url: '', + git_http_url: '', + homepage: '', + ssh_url: '', + http_url: '', + }, + commits: modified_commits, + total_commits_count: 2, + repository: { + name: 'test-repo', + url: 'https://gitlab.com/group1/test-repo1', + description: 'My Cool Project', + homepage: 'https://gitlab.com/group1/test-repo1', + git_http_url: 'https://gitlab.com/group1/test-repo1.git', + git_ssh_url: 'git@gitlab.com:bgroup1/test-repo1.git', + visibility_level: 20, + }, + }, +}; + +/** + * Expected Backstage entities + */ +export const expected_location_entities: MockObject[] = + all_projects_response.map(project => { + const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${project.default_branch}/catalog-info.yaml`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ target: targetUrl, type: 'url' }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }); + +export const expected_added_location_entities: MockObject[] = added_commits.map( + commit => { + const targetUrl = `https://example.com/group1/test-repo1/-/blob/main/${commit.added}`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ target: targetUrl, type: 'url' }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }, +); + +export const expected_removed_location_entities: MockObject[] = + removed_commits.map(commit => { + const targetUrl = `https://example.com/group1/test-repo1/-/blob/main/${commit.removed}`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ target: targetUrl, type: 'url' }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }); + +export const expected_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group1', + 'example.com/team-path': 'group1', + }, + name: 'group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_group_user_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group1', + 'example.com/team-path': 'group1', + }, + name: 'group1', + }, + spec: { + children: [], + members: ['user1'], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_added_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/group-new/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group-new/group1', + 'example.com/team-path': 'group-new/group1', + }, + name: 'group-new-group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_removed_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/group-old/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group-old/group1', + 'example.com/team-path': 'group-old/group1', + }, + name: 'group-old-group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_transformed_group_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group1', + 'example.com/team-path': 'group1', + }, + name: '1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_single_user_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/JohnDoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/JohnDoe', + 'example.com/user-login': 'https://gitlab.example/john_doe', + }, + name: 'JohnDoe', + }, + spec: { + memberOf: [], + profile: { + displayName: 'John Doe', + email: 'john.doe@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_single_user_removed_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/johndoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/johndoe', + 'example.com/user-login': '', + }, + name: 'johndoe', + }, + spec: { + memberOf: [], + profile: { + displayName: 'John Doe', + email: 'john.doe@company.com', + picture: undefined, + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_full_org_scan_entities: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/JohnDoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/JohnDoe', + 'example.com/user-login': 'https://gitlab.example/john_doe', + }, + name: 'JohnDoe', + }, + spec: { + memberOf: ['group1'], + profile: { + displayName: 'John Doe', + email: 'john.doe@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/JaneDoe', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/JaneDoe', + 'example.com/user-login': 'https://gitlab.example/jane_doe', + }, + name: 'JaneDoe', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Jane Doe', + email: 'jane.doe@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/MarySmith', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/MarySmith', + 'example.com/user-login': 'https://gitlab.example/mary_smith', + }, + name: 'MarySmith', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Mary Smith', + email: 'mary.smith@company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://example.com/MarioMario', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/MarioMario', + 'example.com/user-login': 'https://gitlab.example/mario_mario', + }, + name: 'MarioMario', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Mario Mario', + email: 'mario.mario-company.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + + expected_group_entity[0], +]; + +export const expected_full_org_scan_entities_saas: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser1', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser1', + 'gitlab.com/user-login': 'https://gitlab.com/testuser1', + 'gitlab.com/saml-external-uid': '51', + }, + name: 'testuser1', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Test User 1', + email: 'testuser1@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser2', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser2', + 'gitlab.com/user-login': 'https://gitlab.com/testuser2', + 'gitlab.com/saml-external-uid': '52', + }, + name: 'testuser2', + }, + spec: { + memberOf: [], + profile: { + displayName: 'Test User 2', + email: 'testuser2@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 99f6e7869b..9dfe8f8525 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -14,130 +14,25 @@ * limitations under the License. */ -import { ConfigReader } from '@backstage/config'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; -import { readGitLabIntegrationConfig } from '@backstage/integration'; import { getVoidLogger } from '@backstage/backend-common'; -import { graphql, rest } from 'msw'; -import { setupServer, SetupServer } from 'msw/node'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { readGitLabIntegrationConfig } from '@backstage/integration'; +import { setupServer } from 'msw/node'; +import { handlers } from '../__testUtils__/handlers'; +import * as mock from '../__testUtils__/mocks'; import { GitLabClient, paginated } from './client'; import { GitLabGroup, GitLabUser } from './types'; -const server = setupServer(); +const server = setupServer(...handlers); setupRequestMockHandlers(server); -const MOCK_CONFIG = readGitLabIntegrationConfig( - new ConfigReader({ - host: 'example.com', - token: 'test-token', - apiBaseUrl: 'https://example.com/api/v4', - baseUrl: 'https://example.com', - }), -); -const FAKE_PAGED_ENDPOINT = `/some-endpoint`; -const FAKE_PAGED_URL = `${MOCK_CONFIG.apiBaseUrl}${FAKE_PAGED_ENDPOINT}`; - -function setupFakeFourPageURL(srv: SetupServer, url: string) { - srv.use( - rest.get(url, (req, res, ctx) => { - const page = req.url.searchParams.get('page'); - const currentPage = page ? Number(page) : 1; - const fakePageCount = 4; - - return res( - // set next page number header if page requested is less than count - ctx.set( - 'x-next-page', - currentPage < fakePageCount ? String(currentPage + 1) : '', - ), - ctx.json([{ someContentOfPage: currentPage }]), - ); - }), - ); -} - -function setupFakeGroupProjectsEndpoint( - srv: SetupServer, - apiBaseUrl: string, - groupID: string, -) { - srv.use( - rest.get(`${apiBaseUrl}/groups/${groupID}/projects`, (_, res, ctx) => { - return res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - description: 'Project One Description', - name: 'Project One', - path: 'project-one', - }, - ]), - ); - }), - ); -} - -function setupFakeInstanceProjectsEndpoint( - srv: SetupServer, - apiBaseUrl: string, -) { - srv.use( - rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => { - return res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - description: 'Project One Description', - name: 'Project One', - path: 'project-one', - }, - { - id: 2, - description: 'Project Two Description', - name: 'Project Two', - path: 'project-two', - }, - ]), - ); - }), - ); -} - -function setupFakeHasFileEndpoint(srv: SetupServer, apiBaseUrl: string) { - srv.use( - rest.head( - `${apiBaseUrl}/projects/group%2Frepo/repository/files/catalog-info.yaml`, - (req, res, ctx) => { - const branch = req.url.searchParams.get('ref'); - if (branch === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); -} - -function setupFakeURLReturnEndpoint(srv: SetupServer, url: string) { - srv.use( - rest.get(url, (req, res, ctx) => { - return res(ctx.json([{ endpoint: req.url.toString() }])); - }), - ); -} - describe('GitLabClient', () => { describe('isSelfManaged', () => { it('returns true if self managed instance', () => { const client = new GitLabClient({ config: readGitLabIntegrationConfig( - new ConfigReader({ - host: 'example.com', - token: 'test-token', - apiBaseUrl: 'https://example.com/api/v4', - }), + new ConfigReader(mock.config_self_managed), ), logger: getVoidLogger(), }); @@ -145,12 +40,7 @@ describe('GitLabClient', () => { }); it('returns false if gitlab.com', () => { const client = new GitLabClient({ - config: readGitLabIntegrationConfig( - new ConfigReader({ - host: 'gitlab.com', - token: 'test-token', - }), - ), + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); expect(client.isSelfManaged()).toBeFalsy(); @@ -158,31 +48,30 @@ describe('GitLabClient', () => { }); describe('pagedRequest', () => { - beforeEach(() => { - // setup fake paginated endpoint with 4 pages each returning one item - setupFakeFourPageURL(server, FAKE_PAGED_URL); - }); - it('should provide immediate items within the page', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const { items } = await client.pagedRequest(FAKE_PAGED_ENDPOINT); + const { items } = await client.pagedRequest(mock.paged_endpoint); // fake page contains exactly one item expect(items).toHaveLength(1); }); it('should request items for a given page number', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const requestedPage = 2; const { items, nextPage } = await client.pagedRequest( - FAKE_PAGED_ENDPOINT, + mock.paged_endpoint, { page: requestedPage, }, @@ -195,12 +84,14 @@ describe('GitLabClient', () => { it('should not have a next page if at the end', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items, nextPage } = await client.pagedRequest( - FAKE_PAGED_ENDPOINT, + mock.paged_endpoint, { page: 4, }, @@ -211,50 +102,50 @@ describe('GitLabClient', () => { }); it('should throw if response is not okay', async () => { - const endpoint = '/unhealthy-endpoint'; - const url = `${MOCK_CONFIG.apiBaseUrl}${endpoint}`; - server.use( - rest.get(url, (_, res, ctx) => { - return res(ctx.status(400), ctx.json({ error: 'some error' })); - }), - ); - const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); // non-200 status code should throw - await expect(() => client.pagedRequest(endpoint)).rejects.toThrow(); + await expect(() => + client.pagedRequest(mock.unhealthy_endpoint), + ).rejects.toThrow(); }); }); + // TODO: review this. Make it so that the number of projects don't match the results describe('listProjects', () => { it('should get projects for a given group', async () => { - setupFakeGroupProjectsEndpoint( - server, - MOCK_CONFIG.apiBaseUrl, - 'test-group', - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const groupProjectsGen = paginated( options => client.listProjects(options), - { group: 'test-group' }, + { group: 1 }, ); + + const expectedProjects = mock.all_projects_response.filter(project => + project.path_with_namespace!.includes('group1/'), + ); + const allItems = []; for await (const item of groupProjectsGen) { allItems.push(item); } - expect(allItems).toHaveLength(1); + expect(allItems).toHaveLength(expectedProjects.length); }); it('should get all projects for an instance', async () => { - setupFakeInstanceProjectsEndpoint(server, MOCK_CONFIG.apiBaseUrl); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); @@ -266,532 +157,172 @@ describe('GitLabClient', () => { for await (const project of instanceProjects) { allProjects.push(project); } - expect(allProjects).toHaveLength(2); + expect(allProjects).toHaveLength(mock.all_projects_response.length); }); }); - it('listUsers gets all users in the instance', async () => { - server.use( - rest.get(`${MOCK_CONFIG.apiBaseUrl}/users`, (_, res, ctx) => - res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - username: 'test1', - name: 'Test Testit', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.example/test1', - created_at: '2023-01-19T07:27:03.333Z', - bio: '', - location: null, - public_email: null, - skype: '', - linkedin: '', - twitter: '', - website_url: '', - organization: null, - job_title: '', - pronouns: null, - bot: false, - work_information: null, - followers: 0, - following: 0, - is_followed: false, - local_time: null, - last_sign_in_at: '2023-01-19T07:27:49.601Z', - confirmed_at: '2023-01-19T07:27:02.905Z', - last_activity_on: '2023-01-19', - email: 'test@example.com', - theme_id: 1, - color_scheme_id: 1, - projects_limit: 100000, - current_sign_in_at: '2023-01-19T09:09:10.676Z', - identities: [], - can_create_group: true, - can_create_project: true, - two_factor_enabled: false, - external: false, - private_profile: false, - commit_email: 'test@example.com', - is_admin: false, - note: '', - }, - ]), + describe('listUsers', () => { + it('listUsers gets all users in the instance', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), ), - ), - ); - const client = new GitLabClient({ - config: MOCK_CONFIG, - logger: getVoidLogger(), + logger: getVoidLogger(), + }); + + const allUsers: GitLabUser[] = []; + for await (const user of paginated( + options => client.listUsers(options), + {}, + )) { + allUsers.push(user); + } + + expect(allUsers).toMatchObject(mock.all_users_response); }); - - const allUsers: GitLabUser[] = []; - for await (const user of paginated( - options => client.listUsers(options), - {}, - )) { - allUsers.push(user); - } - - expect(allUsers).toMatchObject([ - { - id: 1, - username: 'test1', - email: 'test@example.com', - name: 'Test Testit', - state: 'active', - web_url: 'https://gitlab.example/test1', - avatar_url: 'https://secure.gravatar.com/', - }, - ]); }); - it('listGroups gets all groups in the instance', async () => { - server.use( - rest.get(`${MOCK_CONFIG.apiBaseUrl}/groups`, (_, res, ctx) => - res( - ctx.set('x-next-page', ''), - ctx.json([ - { - id: 1, - web_url: 'https://gitlab.example/groups/group1', - name: 'group1', - path: 'group1', - description: '', - visibility: 'internal', - share_with_group_lock: false, - require_two_factor_authentication: false, - two_factor_grace_period: 48, - project_creation_level: 'developer', - auto_devops_enabled: null, - subgroup_creation_level: 'owner', - emails_disabled: null, - mentions_disabled: null, - lfs_enabled: true, - default_branch_protection: 2, - avatar_url: null, - request_access_enabled: false, - full_name: '8020', - full_path: '8020', - created_at: '2017-06-19T06:42:34.160Z', - parent_id: null, - }, - ]), + describe('listGroups', () => { + it('listGroups gets all groups in the instance', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), ), - ), - ); - const client = new GitLabClient({ - config: MOCK_CONFIG, - logger: getVoidLogger(), + logger: getVoidLogger(), + }); + + const allGroups: GitLabGroup[] = []; + for await (const group of paginated( + options => client.listGroups(options), + {}, + )) { + allGroups.push(group); + } + + expect(allGroups).toMatchObject(mock.all_groups_response); }); - - const allGroups: GitLabGroup[] = []; - for await (const group of paginated( - options => client.listGroups(options), - {}, - )) { - allGroups.push(group); - } - - expect(allGroups).toMatchObject([ - { - id: 1, - name: 'group1', - full_path: '8020', - description: '', - parent_id: null, - }, - ]); }); describe('get gitlab.com users', () => { it('gets all users under group', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: [ - { - user: { - id: 'gid://gitlab/User/1', - username: 'user1', - publicEmail: 'user1@example.com', - name: 'user1', - state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); - const saasMembers = ( - await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + await client.getGroupMembers('saas-multi-user-group', [ + 'DIRECT, DESCENDANTS', + ]) ).items; - const expectedSaasMember = [ - { - id: 1, - username: 'user1', - email: 'user1@example.com', - name: 'user1', - state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', - }, - ]; - expect(saasMembers.length).toEqual(1); - expect(saasMembers).toEqual(expectedSaasMember); + expect(saasMembers.length).toEqual(2); + expect(saasMembers).toEqual(mock.expectedSaasMember); }); - it('gets all users with token without full permissions', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: {}, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); - const saasMembers = ( - await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + await client.getGroupMembers('', ['DIRECT, DESCENDANTS']) ).items; - expect(saasMembers).toEqual([]); }); - it('rejects when GraphQL returns errors', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.errors([ - { message: 'Unexpected end of document', locations: [] }, - ]), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig(new ConfigReader(mock.config_saas)), logger: getVoidLogger(), }); - await expect(() => - client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']), + client.getGroupMembers('error-group', ['DIRECT, DESCENDANTS']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); it('traverses multi-page results', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: req.variables.endCursor - ? [ - { - user: { - id: 'gid://gitlab/User/1', - username: 'user1', - publicEmail: 'user1@example.com', - name: 'user1', - state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', - }, - }, - ] - : [ - { - user: { - id: 'gid://gitlab/User/2', - username: 'user2', - publicEmail: 'user2@example.com', - name: 'user2', - state: 'active', - webUrl: 'user2.com', - avatarUrl: 'user2', - }, - }, - ], - pageInfo: { - endCursor: req.variables.endCursor ? 'end' : 'next', - hasNextPage: !req.variables.endCursor, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const saasMembers = ( - await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) + await client.getGroupMembers('multi-page-saas', ['DIRECT, DESCENDANTS']) ).items; - const expectedSaasMember1 = { - id: 1, - username: 'user1', - email: 'user1@example.com', - name: 'user1', - state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', - }; - - const expectedSaasMember2 = { - id: 2, - username: 'user2', - email: 'user2@example.com', - name: 'user2', - state: 'active', - web_url: 'user2.com', - avatar_url: 'user2', - }; - expect(saasMembers.length).toEqual(2); - expect(saasMembers[0]).toEqual(expectedSaasMember2); - expect(saasMembers[1]).toEqual(expectedSaasMember1); + expect(saasMembers[0]).toEqual(mock.expectedSaasMember[1]); + expect(saasMembers[1]).toEqual(mock.expectedSaasMember[0]); }); }); describe('listDescendantGroups', () => { it('gets all groups under root', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.data({ - group: { - descendantGroups: { - nodes: [ - { - id: 'gid://gitlab/Group/1', - name: 'group1', - description: 'description1', - fullPath: 'path/group1', - parent: { - id: '123', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const saasGroups = (await client.listDescendantGroups('group1')).items; + const allGroups = (await client.listDescendantGroups('group-with-parent')) + .items; - const expectedSaasGroup = [ - { - id: 1, - name: 'group1', - description: 'description1', - full_path: 'path/group1', - parent_id: 123, - }, - ]; - - expect(saasGroups.length).toEqual(1); - expect(saasGroups).toEqual(expectedSaasGroup); + expect(allGroups.length).toEqual(1); + expect(allGroups).toEqual(mock.group_with_parent); }); it('gets all descendant groups with token without full permissions', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.data({ - group: {}, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const saasGroups = (await client.listDescendantGroups('group1')).items; + const allGroups = ( + await client.listDescendantGroups('non-existing-group') + ).items; - expect(saasGroups).toEqual([]); + expect(allGroups).toEqual([]); }); it('rejects when GraphQL returns errors', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.errors([ - { message: 'Unexpected end of document', locations: [] }, - ]), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - await expect(() => client.listDescendantGroups('group1')).rejects.toThrow( + await expect(() => + client.listDescendantGroups('error-group'), + ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); it('traverses multi-page results', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listDescendantGroups', async (req, res, ctx) => - res( - ctx.data({ - group: { - descendantGroups: { - nodes: req.variables.endCursor - ? [ - { - id: 'gid://gitlab/Group/1', - name: 'group1', - description: 'description1', - fullPath: 'path/group1', - parent: { - id: '123', - }, - }, - ] - : [ - { - id: 'gid://gitlab/Group/2', - name: 'group2', - description: 'description2', - fullPath: 'path/group2', - parent: { - id: '123', - }, - }, - ], - pageInfo: { - endCursor: req.variables.endCursor ? 'end' : 'next', - hasNextPage: !req.variables.endCursor, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const saasGroups = (await client.listDescendantGroups('root')).items; - const expectedSaasGroup1 = { - id: 1, - name: 'group1', - description: 'description1', - full_path: 'path/group1', - parent_id: 123, - }; - - const expectedSaasGroup2 = { - id: 2, - name: 'group2', - description: 'description2', - full_path: 'path/group2', - parent_id: 123, - }; - expect(saasGroups.length).toEqual(2); - expect(saasGroups[0]).toEqual(expectedSaasGroup2); - expect(saasGroups[1]).toEqual(expectedSaasGroup1); + expect(saasGroups[0]).toEqual(mock.expectedSaasGroup[1]); + expect(saasGroups[1]).toEqual(mock.expectedSaasGroup[0]); }); }); describe('getGroupMembers', () => { it('gets member IDs', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: [ - { - user: { - id: 'gid://gitlab/User/1', - username: 'user1', - publicEmail: 'user1@example.com', - name: 'user1', - state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); @@ -811,96 +342,144 @@ describe('GitLabClient', () => { }); it('gets member IDs with token without full permissions', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.data({ - group: {}, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', ['DIRECT']); + const members = await client.getGroupMembers('non-existing-group', [ + 'DIRECT', + ]); expect(members.items).toEqual([]); }); + // TODO: is this one really necessary? it('rejects when GraphQL returns errors', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (_, res, ctx) => - res( - ctx.errors([ - { message: 'Unexpected end of document', locations: [] }, - ]), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); await expect(() => - client.getGroupMembers('group1', ['DIRECT']), + client.getGroupMembers('error-group', ['DIRECT']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); it('traverses multi-page results', async () => { - server.use( - graphql - .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('getGroupMembers', async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: req.variables.endCursor - ? [{ user: { id: 'gid://gitlab/User/2' } }] - : [{ user: { id: 'gid://gitlab/User/1' } }], - pageInfo: { - endCursor: req.variables.endCursor ? 'end' : 'next', - hasNextPage: !req.variables.endCursor, - }, - }, - }, - }), - ), - ), - ); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', ['DIRECT']); + const members = await client.getGroupMembers('multi-page', ['DIRECT']); expect(members.items[0].id).toEqual(1); expect(members.items[1].id).toEqual(2); }); }); + + describe('getGroupById', () => { + it('should return group details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + const group = await client.getGroupById(1); + expect(group).toMatchObject(mock.all_groups_response[0]); + }); + + it('should handle errors when fetching group details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + await expect(() => client.getGroupById(42)).rejects.toThrow( + 'Internal Server Error', + ); + }); + }); + + describe('getProjectById', () => { + it('should return project details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + const project = await client.getProjectById(1); + expect(project).toMatchObject(mock.all_projects_response[0]); + }); + + it('should handle errors when fetching project details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + await expect(() => client.getProjectById(42)).rejects.toThrow( + 'Internal Server Error', + ); + }); + }); + + describe('getUserById', () => { + it('should return user details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + const user = await client.getUserById(1); + expect(user).toMatchObject(mock.all_users_response[0]); + }); + + it('should handle errors when fetching user details by ID', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: getVoidLogger(), + }); + + await expect(() => client.getUserById(42)).rejects.toThrow( + 'Internal Server Error', + ); + }); + }); }); describe('paginated', () => { it('should iterate through the pages until exhausted', async () => { - setupFakeFourPageURL(server, FAKE_PAGED_URL); const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const paginatedItems = paginated( - options => client.pagedRequest(FAKE_PAGED_ENDPOINT, options), + options => client.pagedRequest(mock.paged_endpoint, options), {}, ); const allItems = []; @@ -916,25 +495,26 @@ describe('hasFile', () => { let client: GitLabClient; beforeEach(() => { - setupFakeHasFileEndpoint(server, MOCK_CONFIG.apiBaseUrl); client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); }); - it('should not find catalog file', async () => { + it('should find catalog file', async () => { const hasFile = await client.hasFile( - 'group/repo', - 'master', + 'group1/test-repo1', + 'main', 'catalog-info.yaml', ); expect(hasFile).toBe(true); }); - it('should find catalog file', async () => { + it('should not find catalog file', async () => { const hasFile = await client.hasFile( - 'group/repo', + 'group1/test-repo1', 'unknown', 'catalog-info.yaml', ); @@ -943,19 +523,16 @@ describe('hasFile', () => { }); describe('pagedRequest search params', () => { - beforeEach(() => { - // setup fake paginated endpoint with 4 pages each returning one item - setupFakeURLReturnEndpoint(server, FAKE_PAGED_URL); - }); - it('no search params provided', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, ); // fake page contains exactly one item expect(items).toHaveLength(1); @@ -966,12 +543,14 @@ describe('pagedRequest search params', () => { it('defined numeric search params', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, { page: 1, per_page: 50 }, ); // fake page contains exactly one item @@ -985,12 +564,14 @@ describe('pagedRequest search params', () => { it('defined string search params', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, { test: 'value', empty: '' }, ); // fake page contains exactly one item @@ -1004,12 +585,14 @@ describe('pagedRequest search params', () => { it('defined boolean search params', async () => { const client = new GitLabClient({ - config: MOCK_CONFIG, + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), logger: getVoidLogger(), }); const { items } = await client.pagedRequest<{ endpoint: string }>( - FAKE_PAGED_ENDPOINT, + mock.some_endpoint, { active: true, archived: false }, ); // fake page contains exactly one item diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index cc2b2362cd..b30da0461a 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -24,6 +24,7 @@ import { GitLabDescendantGroupsResponse, GitLabGroup, GitLabGroupMembersResponse, + GitLabProject, GitLabUser, PagedResponse, } from './types'; @@ -80,6 +81,39 @@ export class GitLabClient { return this.pagedRequest(`/projects`, options); } + async getProjectById( + projectId: number, + options?: CommonListOptions, + ): Promise { + // Make the request to the GitLab API + const response = await this.nonPagedRequest( + `/projects/${projectId}`, + options, + ); + + return response; + } + + async getGroupById( + groupId: number, + options?: CommonListOptions, + ): Promise { + // Make the request to the GitLab API + const response = await this.nonPagedRequest(`/groups/${groupId}`, options); + + return response; + } + + async getUserById( + userId: number, + options?: CommonListOptions, + ): Promise { + // Make the request to the GitLab API + const response = await this.nonPagedRequest(`/users/${userId}`, options); + + return response; + } + async listUsers( options?: UserListOptions, ): Promise> { @@ -139,7 +173,6 @@ export class GitLabClient { name description fullPath - visibility parent { id } @@ -174,7 +207,6 @@ export class GitLabClient { name: groupItem.name, description: groupItem.description, full_path: groupItem.fullPath, - visibility: groupItem.visibility, parent_id: Number( groupItem.parent.id.replace(/^gid:\/\/gitlab\/Group\//, ''), ), @@ -325,9 +357,13 @@ export class GitLabClient { options?: CommonListOptions, ): Promise> { const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); + for (const key in options) { - if (options[key] !== undefined && options[key] !== '') { - request.searchParams.append(key, options[key]!.toString()); + if (options.hasOwnProperty(key)) { + const value = options[key]; + if (value !== undefined && value !== '') { + request.searchParams.append(key, value.toString()); + } } } @@ -336,6 +372,7 @@ export class GitLabClient { request.toString(), getGitLabRequestOptions(this.config), ); + if (!response.ok) { throw new Error( `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ @@ -343,6 +380,7 @@ export class GitLabClient { } - ${response.statusText}`, ); } + return response.json().then(items => { const nextPage = response.headers.get('x-next-page'); @@ -352,6 +390,37 @@ export class GitLabClient { } as PagedResponse; }); } + + async nonPagedRequest( + endpoint: string, + options?: CommonListOptions, + ): Promise { + const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); + + for (const key in options) { + if (options.hasOwnProperty(key)) { + const value = options[key]; + if (value !== undefined && value !== '') { + request.searchParams.append(key, value.toString()); + } + } + } + + const response = await fetch( + request.toString(), + getGitLabRequestOptions(this.config), + ); + + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + + return response.json(); + } } /** diff --git a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts index 912e5a0502..d7a6bf76e9 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts @@ -49,10 +49,6 @@ export function defaultGroupEntitiesTransformer( const annotations: { [annotationName: string]: string } = {}; annotations[`${options.providerConfig.host}/team-path`] = group.full_path; - if (group.visibility !== undefined) { - annotations[`${options.providerConfig.host}/visibility`] = - group.visibility; - } const entity: GroupEntity = { apiVersion: 'backstage.io/v1alpha1', diff --git a/plugins/catalog-backend-module-gitlab/src/lib/index.ts b/plugins/catalog-backend-module-gitlab/src/lib/index.ts index 1eee268862..cc82bd6a0b 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/index.ts @@ -14,14 +14,15 @@ * limitations under the License. */ +export { readGitlabConfigs } from '../providers/config'; export { GitLabClient, paginated } from './client'; export type { - GitLabUser, GitLabGroup, GitLabGroupSamlIdentity, GitLabProject, - GitlabProviderConfig, + GitLabUser, GitlabGroupDescription, + GitlabProviderConfig, GroupNameTransformer, GroupNameTransformerOptions, GroupTransformer, @@ -29,4 +30,3 @@ export type { UserTransformer, UserTransformerOptions, } from './types'; -export { readGitlabConfigs } from '../providers/config'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 123515ccf3..b60814ade1 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -34,6 +34,9 @@ export type GitlabProjectForkedFrom = { export type GitLabProject = { id: number; + description?: string; + name?: string; + path?: string; default_branch?: string; archived: boolean; last_activity_at: string; @@ -76,7 +79,6 @@ export type GitLabGroup = { name: string; full_path: string; description?: string; - visibility?: string; parent_id?: number; }; @@ -116,7 +118,6 @@ export type GitLabDescendantGroupsResponse = { name: string; description: string; fullPath: string; - visibility: string; parent: { id: string; }; @@ -180,6 +181,11 @@ export type GitlabProviderConfig = { */ groupPattern: RegExp; + /** + * If true, the provider will also ingest add inherited users to the ingested groups + */ + allowInherited?: boolean; + orgEnabled?: boolean; schedule?: TaskScheduleDefinition; /** @@ -242,3 +248,72 @@ export interface GroupTransformerOptions { providerConfig: GitlabProviderConfig; groupNameTransformer: GroupNameTransformer; } + +/** + * Represents the schema for system hook events related to groups. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseGroupEventsSchema = { + created_at: string; + updated_at: string; + name: string; + path: string; + full_path: string; + group_id: number; +}; + +/** + * Represents the schema for system hook events related to users. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseUserEventsSchema = { + created_at: string; + updated_at: string; + email: string; + name: string; + username: string; + user_id: number; +}; + +/** + * Represents the schema for system hook events related to user memberships. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseMembershipEventsSchema = { + created_at: string; + updated_at: string; + group_name: string; + group_path: string; + group_id: number; + user_username: string; + user_email: string; + user_name: string; + user_id: number; + group_access: string; +}; + +/** + * Represents the schema for system hook events related to projects. + * https://docs.gitlab.com/ee/administration/system_hooks.html + * + * @public + */ +export type SystemHookBaseProjectEventsSchema = { + created_at: string; + updated_at: string; + event_name: string; + name: string; + owner_email: string; + owner_name: string; + owners: { name: string; email: string }[]; + path: string; + path_with_namespace: string; + project_id: number; + project_visibility: string; +}; diff --git a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts index 46cfe403e8..3c93cb98f1 100644 --- a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts @@ -14,15 +14,27 @@ * limitations under the License. */ +import { createServiceFactory } from '@backstage/backend-plugin-api'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { TestEventsService } from '@backstage/plugin-events-backend-test-utils'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; import { Duration } from 'luxon'; -import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider'; import { GitlabDiscoveryEntityProvider } from '../providers'; +import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider'; describe('catalogModuleGitlabDiscoveryEntityProvider', () => { it('should register provider at the catalog extension point', async () => { + const events = new TestEventsService(); + const eventsServiceFactory = createServiceFactory({ + service: eventsServiceRef, + deps: {}, + async factory({}) { + return events; + }, + }); let addedProviders: Array | undefined; let usedSchedule: TaskScheduleDefinition | undefined; @@ -31,6 +43,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => { addedProviders = providers; }, }; + const connection = jest.fn() as unknown as EntityProviderConnection; const runner = jest.fn(); const scheduler = mockServices.scheduler.mock({ createScheduledTaskRunner(schedule) { @@ -68,6 +81,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => { await startTestBackend({ extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], features: [ + eventsServiceFactory(), catalogModuleGitlabDiscoveryEntityProvider(), mockServices.rootConfig.factory({ data: config }), mockServices.logger.factory(), @@ -78,9 +92,17 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => { expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M')); expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M')); expect(addedProviders?.length).toEqual(1); - expect(addedProviders?.pop()?.getProviderName()).toEqual( + expect(runner).not.toHaveBeenCalled(); + + const provider = addedProviders!.pop()!; + expect(provider.getProviderName()).toEqual( 'GitlabDiscoveryEntityProvider:test-id', ); - expect(runner).not.toHaveBeenCalled(); + await provider.connect(connection); + expect(events.subscribed).toHaveLength(1); + expect(events.subscribed[0].id).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + expect(runner).toHaveBeenCalledTimes(1); }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts index a1b24e4eea..bba8348a74 100644 --- a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts @@ -19,6 +19,7 @@ import { createBackendModule, } from '@backstage/backend-plugin-api'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; import { GitlabDiscoveryEntityProvider } from '../providers'; /** @@ -26,6 +27,7 @@ import { GitlabDiscoveryEntityProvider } from '../providers'; * * @alpha */ + export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({ pluginId: 'catalog', moduleId: 'gitlab-discovery-entity-provider', @@ -36,14 +38,16 @@ export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({ catalog: catalogProcessingExtensionPoint, logger: coreServices.logger, scheduler: coreServices.scheduler, + events: eventsServiceRef, }, - async init({ config, catalog, logger, scheduler }) { - catalog.addEntityProvider( + async init({ config, catalog, logger, scheduler, events }) { + const gitlabDiscoveryEntityProvider = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, + events, scheduler, - }), - ); + }); + catalog.addEntityProvider(gitlabDiscoveryEntityProvider); }, }); }, diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index 7508b35b84..bd725ee0ce 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -23,10 +23,16 @@ import { import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; -import { rest } from 'msw'; +import { DefaultEventsService } from '@backstage/plugin-events-node'; import { setupServer } from 'msw/node'; +import { handlers } from '../__testUtils__/handlers'; +import * as mock from '../__testUtils__/mocks'; import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; +const server = setupServer(...handlers); +setupRequestMockHandlers(server); +afterEach(() => jest.resetAllMocks()); + class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -42,13 +48,8 @@ class PersistingTaskRunner implements TaskRunner { const logger = getVoidLogger(); -const server = setupServer(); - -describe('GitlabDiscoveryEntityProvider', () => { - setupRequestMockHandlers(server); - afterEach(() => jest.resetAllMocks()); - - it('no provider config', () => { +describe('GitlabDiscoveryEntityProvider - configuration', () => { + it('should not instantiate providers when no config found', () => { const schedule = new PersistingTaskRunner(); const config = new ConfigReader({}); const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { @@ -59,29 +60,46 @@ describe('GitlabDiscoveryEntityProvider', () => { expect(providers).toHaveLength(0); }); - it('single simple discovery config', () => { + it('should fail without schedule nor scheduler', () => { + const config = new ConfigReader(mock.config_single_integration_branch); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('should fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader(mock.config_no_schedule_integration); + + expect(() => + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id', + ); + }); + it('should throw error when no matching GitLab integration config found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_non_gitlab_host); + + expect(() => { + GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow('No gitlab integration found that matches host example.com'); + }); + + it('should instantiate provider with single simple discovery config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader(mock.config_single_integration_branch); const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -93,33 +111,9 @@ describe('GitlabDiscoveryEntityProvider', () => { ); }); - it('multiple discovery configs', () => { + it('should instantiate providers when multiple discovery configs', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - 'second-test': { - host: 'test-gitlab', - group: 'second-group', - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_double_integration); const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -133,29 +127,10 @@ describe('GitlabDiscoveryEntityProvider', () => { 'GitlabDiscoveryEntityProvider:second-test', ); }); - - it('apply full update on scheduled execution', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); +}); +describe('GitlabDiscoveryEntityProvider - refresh', () => { + it('should apply full update on scheduled execution', async () => { + const config = new ConfigReader(mock.config_no_org_integration); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -169,92 +144,28 @@ describe('GitlabDiscoveryEntityProvider', () => { 'GitlabDiscoveryEntityProvider:test-id', ); - server.use( - rest.get( - `https://api.gitlab.example/api/v4/groups/test-group/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); - await provider.connect(entityProviderConnection); const taskDef = schedule.getTasks()[0]; expect(taskDef.id).toEqual('GitlabDiscoveryEntityProvider:test-id:refresh'); await (taskDef.fn as () => Promise)(); - const url = `https://api.gitlab.example/test-group/test-repo/-/blob/master/catalog-info.yaml`; - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': `url:${url}`, - 'backstage.io/managed-by-origin-location': `url:${url}`, - }, - name: 'generated-cd37bf72a2fe92603f4255d9f49c6c1ead746a48', - }, - spec: { - presence: 'optional', - target: `${url}`, - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ]; - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: expectedEntities, + entities: mock.expected_location_entities.filter( + entity => + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), }); }); it('should filter found projects based on a provided project pattern', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - projectPattern: 'john/', - }, - }, - }, - }, - }); + const config = new ConfigReader( + mock.config_single_integration_project_pattern, + ); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -265,50 +176,10 @@ describe('GitlabDiscoveryEntityProvider', () => { schedule, })[0]; - server.use( - rest.get( - `https://api.gitlab.example/api/v4/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - { - id: 124, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/john/example', - path_with_namespace: 'john/example', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); + const projectPattern = + mock.config_single_integration_project_pattern.catalog.providers.gitlab[ + 'test-id' + ].projectPattern; await provider.connect(entityProviderConnection); @@ -316,55 +187,20 @@ describe('GitlabDiscoveryEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - 'backstage.io/managed-by-origin-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - }, - name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9', - }, - spec: { - presence: 'optional', - target: - 'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ], + entities: mock.expected_location_entities.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(projectPattern) && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), }); }); it('should filter fork projects', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - skipForkedRepos: true, - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_single_integration_skip_forks); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -375,216 +211,26 @@ describe('GitlabDiscoveryEntityProvider', () => { schedule, })[0]; - server.use( - rest.get( - `https://api.gitlab.example/api/v4/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - forked_from_project: { - id: 13083, - }, - }, - { - id: 124, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/john/example', - path_with_namespace: 'john/example', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'master') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); - await provider.connect(entityProviderConnection); await provider.refresh(logger); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - 'backstage.io/managed-by-origin-location': - 'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - }, - name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9', - }, - spec: { - presence: 'optional', - target: - 'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml', - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ], + entities: mock.expected_location_entities.filter( + entity => + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('forked') && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), }); }); - it('fail without schedule and scheduler', () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); - - expect(() => - GitlabDiscoveryEntityProvider.fromConfig(config, { - logger, - }), - ).toThrow('Either schedule or scheduler must be provided'); - }); - - it('fail with scheduler but no schedule config', () => { - const scheduler = { - createScheduledTaskRunner: (_: any) => jest.fn(), - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); - - expect(() => - GitlabDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ).toThrow( - 'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id', - ); - }); - - it('single simple provider config with schedule in config', async () => { - const schedule = new PersistingTaskRunner(); - const scheduler = { - createScheduledTaskRunner: (_: any) => schedule, - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - schedule: { - frequency: 'PT30M', - timeout: 'PT3M', - }, - }, - }, - }, - }, - }); - const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }); - - expect(providers).toHaveLength(1); - expect(providers[0].getProviderName()).toEqual( - 'GitlabDiscoveryEntityProvider:test-id', - ); - }); - it('should filter found projects based on the branch', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - branch: 'test', - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_single_integration_branch); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -595,47 +241,9 @@ describe('GitlabDiscoveryEntityProvider', () => { schedule, })[0]; - server.use( - rest.get( - `https://api.gitlab.example/api/v4/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - { - id: 124, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/john/example', - path_with_namespace: 'john/example', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', - (_, res, ctx) => { - return res(ctx.status(404, 'Not Found')); - }, - ), - rest.head( - 'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml', - (req, res, ctx) => { - if (req.url.searchParams.get('ref') === 'test') { - return res(ctx.status(200)); - } - return res(ctx.status(404, 'Not Found')); - }, - ), - ); + const configured_branch = + mock.config_single_integration_branch.catalog.providers.gitlab['test-id'] + .branch; await provider.connect(entityProviderConnection); @@ -643,30 +251,229 @@ describe('GitlabDiscoveryEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml', - 'backstage.io/managed-by-origin-location': - 'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml', - }, - name: 'generated-232185d858fee049986d202c10316d634e76a3d1', - }, - spec: { - presence: 'optional', - target: - 'https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml', - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }, - ], + entities: mock.expected_location_entities.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(configured_branch) && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), + }); + }); + + it('should only include projects with fallback branch', async () => { + const config = new ConfigReader(mock.config_fallbackBranch_branch); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const configured_branch = + mock.config_fallbackBranch_branch.catalog.providers.gitlab['test-id'] + .fallbackBranch; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities.filter( + entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(configured_branch) && + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), + }); + }); + + it('should ignore projects outside group scope', async () => { + const config = new ConfigReader(mock.config_single_integration_group); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const configured_group = + mock.config_single_integration_group.catalog.providers.gitlab['test-id'] + .group; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities.filter(entity => + entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes(configured_group), + ), }); }); }); +describe('GitlabDiscoveryEntityProvider - events', () => { + it('should ignore push event if project is forked', async () => { + const config = new ConfigReader(mock.config_single_integration_skip_forks); + + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event_forked); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it(`should skip refresh and mutation when project pattern doesn't match`, async () => { + const config = new ConfigReader(mock.config_unmatched_project_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should ignore projects outside group scope', async () => { + const config = new ConfigReader(mock.config_single_integration_group); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event_unmatched_group); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + it('should apply delta mutations on added files from push event', async () => { + const config = new ConfigReader(mock.config_single_integration_branch); + + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_add_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_added_location_entities, + removed: [], + }); + }); + + it('should apply delta mutations on removed files from push event', async () => { + const config = new ConfigReader(mock.config_single_integration_branch); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + await events.publish(mock.push_remove_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: mock.expected_removed_location_entities, + }); + }); + + it('should call refresh on added files from push event', async () => { + const config = new ConfigReader(mock.config_single_integration_branch); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + await provider.connect(entityProviderConnection); + + const url = `https://example.com/group1/test-repo1`; + + await events.publish(mock.push_modif_event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.refresh).toHaveBeenCalledWith({ + keys: [ + `url:${url}/-/tree/main/catalog-info.yaml`, + `url:${url}/-/blob/main/catalog-info.yaml`, + ], + }); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + // EventSupportChange: stop add tests >>> +}); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index ae8fe4e12c..653b12060e 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -17,12 +17,15 @@ import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, - LocationSpec, locationSpecToLocationEntity, } from '@backstage/plugin-catalog-node'; +import { EventsService } from '@backstage/plugin-events-node'; +import { WebhookProjectSchema, WebhookPushEventSchema } from '@gitbeaker/rest'; import * as uuid from 'uuid'; import { GitLabClient, @@ -33,26 +36,37 @@ import { } from '../lib'; import { LoggerService } from '@backstage/backend-plugin-api'; +import * as path from 'path'; + +const TOPIC_REPO_PUSH = 'gitlab.push'; + type Result = { scanned: number; matches: GitLabProject[]; }; /** - * Discovers entity definition files in the groups of a Gitlab instance. + * Discovers catalog files located in your GitLab instance. + * The provider will search your GitLab instance's projects and register catalog files matching the configured path + * as Location entity and via following processing steps add all contained catalog entities. + * This can be useful as an alternative to static locations or manually adding things to the catalog. + * * @public */ +// <<< EventSupportChange: implemented EventSubscriber interface export class GitlabDiscoveryEntityProvider implements EntityProvider { private readonly config: GitlabProviderConfig; private readonly integration: GitLabIntegration; private readonly logger: LoggerService; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; + private readonly events?: EventsService; static fromConfig( config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; }, @@ -92,13 +106,20 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }), ); }); + return providers; } + /** + * Constructs a GitlabDiscoveryEntityProvider instance. + * + * @param options - Configuration options including config, integration, logger, and taskRunner. + */ private constructor(options: { config: GitlabProviderConfig; integration: GitLabIntegration; logger: LoggerService; + events?: EventsService; taskRunner: TaskRunner; }) { this.config = options.config; @@ -107,6 +128,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { target: this.getProviderName(), }); this.scheduleFn = this.createScheduleFn(options.taskRunner); + this.events = options.events; } getProviderName(): string { @@ -116,8 +138,28 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { async connect(connection: EntityProviderConnection): Promise { this.connection = connection; await this.scheduleFn(); + + if (this.events) { + await this.events.subscribe({ + id: this.getProviderName(), + topics: [TOPIC_REPO_PUSH], + onEvent: async params => { + if (params.topic !== TOPIC_REPO_PUSH) { + return; + } + + await this.onRepoPush(params.eventPayload as WebhookPushEventSchema); + }, + }); + } } + /** + * Creates a scheduled task runner for refreshing the entity provider. + * + * @param taskRunner - The task runner instance. + * @returns The scheduled function. + */ private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; @@ -143,13 +185,17 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }; } + /** + * Performs a full scan on the GitLab instance searching for locations to be ingested + * + * @param logger - The logger instance for logging. + */ async refresh(logger: LoggerService): Promise { if (!this.connection) { throw new Error( `Gitlab discovery connection not initialized for ${this.getProviderName()}`, ); } - const client = new GitLabClient({ config: this.integration.config, logger: logger, @@ -171,38 +217,8 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }; for await (const project of projects) { - if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) { - continue; - } - - res.scanned++; - - if ( - this.config.skipForkedRepos && - project.hasOwnProperty('forked_from_project') - ) { - continue; - } - - if ( - !this.config.branch && - this.config.fallbackBranch === '*' && - project.default_branch === undefined - ) { - continue; - } - - const project_branch = - this.config.branch ?? - project.default_branch ?? - this.config.fallbackBranch; - - const projectHasFile: boolean = await client.hasFile( - project.path_with_namespace ?? '', - project_branch, - this.config.catalogFile, - ); - if (projectHasFile) { + if (await this.shouldProcessProject(project, client)) { + res.scanned++; res.matches.push(project); } } @@ -222,10 +238,255 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { this.config.branch ?? project.default_branch ?? this.config.fallbackBranch; + return { type: 'url', target: `${project.web_url}/-/blob/${project_branch}/${this.config.catalogFile}`, presence: 'optional', }; } + + /** + * Handles the "gitlab.push" event. + * + * @param event - The push event payload. + */ + private async onRepoPush(event: WebhookPushEventSchema): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + this.logger.info( + `Received push event for ${event.project.path_with_namespace}`, + ); + + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + const project = await client.getProjectById(event.project_id); + + if (!project) { + this.logger.debug( + `Ignoring push event for ${event.project.path_with_namespace}`, + ); + + return; + } + + if (!(await this.shouldProcessProject(project, client))) { + this.logger.debug(`Skipping event ${event.project.path_with_namespace}`); + return; + } + + // Get array of added, removed or modified files from the push event + const added = this.getFilesMatchingConfig( + event, + 'added', + this.config.catalogFile, + ); + const removed = this.getFilesMatchingConfig( + event, + 'removed', + this.config.catalogFile, + ); + const modified = this.getFilesMatchingConfig( + event, + 'modified', + this.config.catalogFile, + ); + + // Modified files will be scheduled to a refresh + const addedEntities = this.createLocationSpecCommitedFiles( + event.project, + added, + ); + + const removedEntities = this.createLocationSpecCommitedFiles( + event.project, + removed, + ); + + if (addedEntities.length > 0 || removedEntities.length > 0) { + await this.connection.applyMutation({ + type: 'delta', + added: this.toDeferredEntities( + addedEntities.map(entity => entity.target), + ), + removed: this.toDeferredEntities( + removedEntities.map(entity => entity.target), + ), + }); + } + + if (modified.length > 0) { + const projectBranch = + this.config.branch ?? + event.project.default_branch ?? + this.config.fallbackBranch; + + // scheduling a refresh to both tree and blob (https://git-scm.com/book/en/v2/Git-Internals-Git-Objects) + await this.connection.refresh({ + keys: [ + ...modified.map( + filePath => + `url:${event.project.web_url}/-/tree/${projectBranch}/${filePath}`, + ), + ...modified.map( + filePath => + `url:${event.project.web_url}/-/blob/${projectBranch}/${filePath}`, + ), + ], + }); + } + + this.logger.info( + `Processed GitLab push event from ${event.project.web_url}: added ${added.length} - removed ${removed.length} - modified ${modified.length}`, + ); + } + + /** + * Gets files matching the specified commit action and catalog file name. + * + * @param event - The push event payload. + * @param action - The action type ('added', 'removed', or 'modified'). + * @param catalogFile - The catalog file name. + * @returns An array of file paths. + */ + private getFilesMatchingConfig( + event: WebhookPushEventSchema, + action: 'added' | 'removed' | 'modified', + catalogFile: string, + ): string[] { + if (!event.commits) { + return []; + } + + const matchingFiles = event.commits.flatMap((element: any) => + element[action].filter( + (file: string) => path.basename(file) === catalogFile, + ), + ); + + if (matchingFiles.length === 0) { + this.logger.debug( + `No files matching '${catalogFile}' found in the commits.`, + ); + } + + return matchingFiles; + } + + /** + * Creates Backstage location specs for committed files. + * + * @param project - The GitLab project information. + * @param addedFiles - The array of added file paths. + * @returns An array of location specs. + */ + private createLocationSpecCommitedFiles( + project: WebhookProjectSchema, + addedFiles: string[], + ): LocationSpec[] { + const projectBranch = + this.config.branch ?? + project.default_branch ?? + this.config.fallbackBranch; + + // Filter added files that match the catalog file pattern + const matchingFiles = addedFiles.filter( + file => path.basename(file) === this.config.catalogFile, + ); + + // Create a location spec for each matching file + const locationSpecs: LocationSpec[] = matchingFiles.map(file => ({ + type: 'url', + target: `${project.web_url}/-/blob/${projectBranch}/${file}`, + presence: 'optional', + })); + + return locationSpecs; + } + + /** + * Converts a target URL to a LocationSpec object. + * + * @param {string} target - The target URL to be converted. + * @returns {LocationSpec} The LocationSpec object representing the URL. + */ + private toLocationSpec(target: string): LocationSpec { + return { + type: 'url', + target: target, + presence: 'optional', + }; + } + + private toDeferredEntities(targets: string[]): DeferredEntity[] { + return targets + .map(target => { + const location = this.toLocationSpec(target); + + return locationSpecToLocationEntity({ location }); + }) + .map(entity => { + return { + locationKey: this.getProviderName(), + entity: entity, + }; + }); + } + + private async shouldProcessProject( + project: GitLabProject, + client: GitLabClient, + ): Promise { + if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it does not match the project pattern ${this.config.projectPattern}.`, + ); + return false; + } + + if ( + this.config.group && + !project.path_with_namespace!.startsWith(`${this.config.group}/`) + ) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it does not match the group pattern ${this.config.group}.`, + ); + return false; + } + + if ( + this.config.skipForkedRepos && + project.hasOwnProperty('forked_from_project') + ) { + this.logger.debug( + `Skipping project ${project.path_with_namespace} as it is a forked project.`, + ); + return false; + } + + const customFallbackBranch = + this.config.fallbackBranch !== 'master' + ? this.config.fallbackBranch + : undefined; + + const project_branch = + this.config.branch ?? + customFallbackBranch ?? + project.default_branch ?? + this.config.fallbackBranch; + + const hasFile = await client.hasFile( + project.path_with_namespace ?? '', + project_branch, + this.config.catalogFile, + ); + + return hasFile; + } } diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 8dffdf1d7b..a65b0a1bc2 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -23,10 +23,17 @@ import { import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; -import { graphql, rest } from 'msw'; +import { DefaultEventsService } from '@backstage/plugin-events-node'; import { setupServer } from 'msw/node'; +import { handlers } from '../__testUtils__/handlers'; +import * as mock from '../__testUtils__/mocks'; +import { GroupNameTransformerOptions } from '../lib/types'; import { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider'; +const server = setupServer(...handlers); +setupRequestMockHandlers(server); +afterEach(() => jest.resetAllMocks()); + class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -42,13 +49,8 @@ class PersistingTaskRunner implements TaskRunner { const logger = getVoidLogger(); -const server = setupServer(); - -describe('GitlabOrgDiscoveryEntityProvider', () => { - setupRequestMockHandlers(server); - afterEach(() => jest.resetAllMocks()); - - it('no provider config', () => { +describe('GitlabOrgDiscoveryEntityProvider - configuration', () => { + it('should not instantiate providers when no config found', () => { const schedule = new PersistingTaskRunner(); const config = new ConfigReader({}); const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { @@ -59,61 +61,89 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { expect(providers).toHaveLength(0); }); - it('single simple discovery config with org disabled', () => { + it('should throw error when no matching GitLab integration config found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - }, - }, - }, - }, - }); - const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - schedule, - }); + const config = new ConfigReader(mock.config_non_gitlab_host); - expect(providers).toHaveLength(0); + expect(() => { + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow('No gitlab integration found that matches host example.com'); }); - it('single simple discovery config with org enabled', () => { + it('should throw error when org configuration not found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - }, - }, - }, + const config = new ConfigReader(mock.config_no_org_integration); + + expect(() => { + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow('Org not enabled for test-id'); + }); + + it('should throw error when saas without group configuration', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader(mock.config_saas_no_group); + + expect(() => { + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + }).toThrow( + `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`, + ); + }); + it('should fail without schedule and scheduler', () => { + const config = new ConfigReader(mock.config_org_integration_saas); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('should fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader(mock.config_org_integration_saas); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:test-id', + ); + }); + + it('should instantiate provider with single simple saas config', async () => { + const schedule = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + const config = new ConfigReader(mock.config_org_integration_saas_sched); + const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + }); + + it('should instantiate provider when org enabled', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader(mock.config_org_integration); const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -125,35 +155,9 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); }); - it('multiple discovery configs', () => { + it('should instantiate providers when multiple valid provider configs', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - 'second-test': { - host: 'test-gitlab', - group: 'second-group', - orgEnabled: true, - }, - }, - }, - }, - }); + const config = new ConfigReader(mock.config_org_double_integration); const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { logger, schedule, @@ -167,29 +171,11 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { 'GitlabOrgDiscoveryEntityProvider:second-test', ); }); +}); - it('apply full update on scheduled execution', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - orgEnabled: true, - }, - }, - }, - }, - }); +describe('GitlabOrgDiscoveryEntityProvider - refresh', () => { + it('should apply full update on scheduled execution', async () => { + const config = new ConfigReader(mock.config_org_integration); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -203,185 +189,6 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { 'GitlabOrgDiscoveryEntityProvider:test-id', ); - server.use( - rest.get( - `https://api.gitlab.example/api/v4/groups/test-group/projects`, - (_req, res, ctx) => { - const response = [ - { - id: 123, - default_branch: 'master', - archived: false, - last_activity_at: new Date().toString(), - web_url: 'https://api.gitlab.example/test-group/test-repo', - path_with_namespace: 'test-group/test-repo', - }, - ]; - return res(ctx.json(response)); - }, - ), - rest.get(`https://api.gitlab.example/api/v4/users`, (_req, res, ctx) => { - const response = [ - { - id: 1, - username: 'test1', - name: 'Test Testit', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.example/test1', - created_at: '2023-01-19T07:27:03.333Z', - bio: '', - location: null, - public_email: null, - skype: '', - linkedin: '', - twitter: '', - website_url: '', - organization: null, - job_title: '', - pronouns: null, - bot: false, - work_information: null, - followers: 0, - following: 0, - is_followed: false, - local_time: null, - last_sign_in_at: '2023-01-19T07:27:49.601Z', - confirmed_at: '2023-01-19T07:27:02.905Z', - last_activity_on: '2023-01-19', - email: 'test@example.com', - theme_id: 1, - color_scheme_id: 1, - projects_limit: 100000, - current_sign_in_at: '2023-01-19T09:09:10.676Z', - identities: [], - can_create_group: true, - can_create_project: true, - two_factor_enabled: false, - external: false, - private_profile: false, - commit_email: 'test@example.com', - is_admin: false, - note: '', - }, - { - id: 2, - username: 'test2', - name: '', - state: 'active', - avatar_url: '', - web_url: 'https://gitlab.example/test2', - created_at: '2023-01-19T07:27:03.333Z', - bio: '', - location: null, - public_email: null, - skype: '', - linkedin: '', - twitter: '', - website_url: '', - organization: null, - job_title: '', - pronouns: null, - bot: false, - work_information: null, - followers: 0, - following: 0, - is_followed: false, - local_time: null, - last_sign_in_at: '2023-01-19T07:27:49.601Z', - confirmed_at: '2023-01-19T07:27:02.905Z', - last_activity_on: '2023-01-19', - email: 'test@example.com', - theme_id: 1, - color_scheme_id: 1, - projects_limit: 100000, - current_sign_in_at: '2023-01-19T09:09:10.676Z', - identities: [], - can_create_group: true, - can_create_project: true, - two_factor_enabled: false, - external: false, - private_profile: false, - commit_email: 'test@example.com', - is_admin: false, - note: '', - }, - ]; - return res(ctx.json(response)); - }), - rest.get(`https://api.gitlab.example/api/v4/groups`, (_req, res, ctx) => { - const response = [ - { - id: 1, - web_url: 'https://gitlab.example/groups/group1', - name: 'group1', - path: 'group1', - description: '', - visibility: 'internal', - share_with_group_lock: false, - require_two_factor_authentication: false, - two_factor_grace_period: 48, - project_creation_level: 'developer', - auto_devops_enabled: null, - subgroup_creation_level: 'owner', - emails_disabled: null, - mentions_disabled: null, - lfs_enabled: true, - default_branch_protection: 2, - avatar_url: null, - request_access_enabled: false, - full_name: '8020', - full_path: '8020', - created_at: '2017-06-19T06:42:34.160Z', - parent_id: null, - }, - { - id: 2, - web_url: 'https://gitlab.example/groups/group1/group2', - name: 'group2', - path: 'group1/group2', - description: 'Group2', - visibility: 'internal', - share_with_group_lock: false, - require_two_factor_authentication: false, - two_factor_grace_period: 48, - project_creation_level: 'developer', - auto_devops_enabled: null, - subgroup_creation_level: 'owner', - emails_disabled: null, - mentions_disabled: null, - lfs_enabled: true, - request_access_enabled: false, - full_name: 'group2', - full_path: 'group1/group2', - created_at: '2017-12-07T13:20:40.675Z', - parent_id: null, - }, - ]; - return res(ctx.json(response)); - }), - graphql - .link('https://test-gitlab/api/graphql') - .operation(async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: - req.variables.group === 'group1/group2' - ? [{ user: { id: 'gid://gitlab/User/1' } }] - : [], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - ); - await provider.connect(entityProviderConnection); const taskDef = schedule.getTasks()[0]; @@ -390,115 +197,15 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); await (taskDef.fn as () => Promise)(); - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://test-gitlab/test1', - 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/test1', - 'test-gitlab/user-login': 'https://gitlab.example/test1', - }, - name: 'test1', - }, - spec: { - memberOf: ['group1-group2'], - profile: { - displayName: 'Test Testit', - email: 'test@example.com', - picture: 'https://secure.gravatar.com/', - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://test-gitlab/test2', - 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/test2', - 'test-gitlab/user-login': 'https://gitlab.example/test2', - }, - name: 'test2', - }, - spec: { - memberOf: [], - profile: { - displayName: undefined, - email: 'test@example.com', - picture: undefined, - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://test-gitlab/group1/group2', - 'backstage.io/managed-by-origin-location': - 'url:https://test-gitlab/group1/group2', - 'test-gitlab/team-path': 'group1/group2', - 'test-gitlab/visibility': 'internal', - }, - description: 'Group2', - name: 'group1-group2', - }, - spec: { - children: [], - profile: { - displayName: 'group2', - }, - type: 'team', - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - ]; - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: expectedEntities, + entities: mock.expected_full_org_scan_entities, }); }); - it('apply full update on scheduled execution for gitlab.com', async () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'gitlab.com', - apiBaseUrl: 'https://gitlab.com/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'gitlab.com', - group: 'group1', - orgEnabled: true, - }, - }, - }, - }, - }); + it('should exclude inactive user', async () => { + const config = new ConfigReader(mock.config_org_integration); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -512,126 +219,70 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { 'GitlabOrgDiscoveryEntityProvider:test-id', ); - server.use( - graphql - .link('https://gitlab.com/api/graphql') - .query('listDescendantGroups', async (_, res, ctx) => - res( - ctx.data({ - group: { - descendantGroups: { - nodes: [ - { - id: 'gid://gitlab/Group/456', - name: 'group2', - description: 'Group2', - fullPath: 'group1/group2', - visibility: 'private', - parent: { - id: 'gid://gitlab/Group/123', - }, - }, - { - id: 'gid://gitlab/Group/789', - name: 'group3', - description: 'Group3', - fullPath: 'group1/group3', - visibility: 'public', - parent: { - id: 'gid://gitlab/Group/123', - }, - }, - ], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - graphql - .link('https://gitlab.com/api/graphql') - .query('getGroupMembers', async (req, res, ctx) => - res( - ctx.data({ - group: { - groupMembers: { - nodes: - req.variables.group === 'group1/group2' - ? [{ user: { id: 'gid://gitlab/User/12' } }] - : [{ user: { id: 'gid://gitlab/User/34' } }], - pageInfo: { - endCursor: 'end', - hasNextPage: false, - }, - }, - }, - }), - ), - ), - rest.get( - `https://gitlab.com/api/v4/groups/group1/members/all`, - (_req, res, ctx) => { - const response = [ - { - access_level: 30, - created_at: '2023-07-17T08:58:34.984Z', - expires_at: null, - id: 12, - username: 'testuser1', - name: 'Test User 1', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.com/testuser1', - email: 'testuser1@example.com', - group_saml_identity: { - provider: 'group_saml', - extern_uid: '51', - saml_provider_id: 1, - }, - is_using_seat: true, - membership_state: 'active', - }, - { - access_level: 30, - created_at: '2023-07-19T08:58:34.984Z', - expires_at: null, - id: 34, - username: 'testuser2', - name: 'Test User 2', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: 'https://gitlab.com/testuser2', - email: 'testuser2@example.com', - group_saml_identity: { - provider: 'group_saml', - extern_uid: '52', - saml_provider_id: 1, - }, - is_using_seat: true, - membership_state: 'active', - }, - { - access_level: 50, - created_at: '2023-07-15T08:58:34.984Z', - expires_at: '2023-10-26', - id: 54, - username: 'group_100_bot_23dc8057bef66e05181f39be4652577c', - name: 'Token Bot', - state: 'active', - avatar_url: 'https://secure.gravatar.com/', - web_url: - 'https://gitlab.com/group_100_bot_23dc8057bef66e05181f39be4652577c', - group_saml_identity: null, - is_using_seat: false, - membership_state: 'active', - }, - ]; - return res(ctx.json(response)); - }, - ), + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', + ); + await (taskDef.fn as () => Promise)(); + const userEntities = mock.expected_full_org_scan_entities.filter( + entity => entity.entity.kind === 'User', + ); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(userEntities).not.toHaveLength(mock.all_users_response.length); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_full_org_scan_entities, + }); + }); + + it('should exclude user when email or username does not match userPattern', async () => { + const config = new ConfigReader(mock.config_userPattern_integration); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + + await (taskDef.fn as () => Promise)(); + const userEntities = mock.expected_full_org_scan_entities.filter( + element => element.entity.metadata.name !== 'MarioMario', + ); // filter out user with non matched e-mail + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(userEntities).not.toHaveLength(mock.all_users_response.length); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: userEntities, + }); + }); + + it('should apply full update on scheduled execution for gitlab.com', async () => { + const config = new ConfigReader(mock.config_org_integration_saas); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', ); await provider.connect(entityProviderConnection); @@ -642,263 +293,340 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); await (taskDef.fn as () => Promise)(); - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/testuser1', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/testuser1', - 'gitlab.com/user-login': 'https://gitlab.com/testuser1', - 'gitlab.com/saml-external-uid': '51', - }, - name: 'testuser1', - }, - spec: { - memberOf: ['group2'], - profile: { - displayName: 'Test User 1', - email: 'testuser1@example.com', - picture: 'https://secure.gravatar.com/', - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/testuser2', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/testuser2', - 'gitlab.com/user-login': 'https://gitlab.com/testuser2', - 'gitlab.com/saml-external-uid': '52', - }, - name: 'testuser2', - }, - spec: { - memberOf: ['group3'], - profile: { - displayName: 'Test User 2', - email: 'testuser2@example.com', - picture: 'https://secure.gravatar.com/', - }, - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/group1/group2', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/group1/group2', - 'gitlab.com/team-path': 'group1/group2', - 'gitlab.com/visibility': 'private', - }, - description: 'Group2', - name: 'group2', - }, - spec: { - children: [], - profile: { - displayName: 'group2', - }, - type: 'team', - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://gitlab.com/group1/group3', - 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/group1/group3', - 'gitlab.com/team-path': 'group1/group3', - 'gitlab.com/visibility': 'public', - }, - description: 'Group3', - name: 'group3', - }, - spec: { - children: [], - profile: { - displayName: 'group3', - }, - type: 'team', - }, - }, - locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - }, - ]; - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'full', - entities: expectedEntities, + entities: mock.expected_full_org_scan_entities_saas, + }); + }); +}); + +describe('GitlabOrgDiscoveryEntityProvider with events support', () => { + it('should ignore gitlab.group_destroy event when group pattern does not match', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_destroy_event_unmatched); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should apply a delta mutation if gitlab.group_destroy event received and defaultGroupTransformation in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_destroy_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: mock.expected_group_entity, + }); + }); + + it('should ignore gitlab.group_create event when group pattern does not match', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + await events.publish(mock.group_create_event_unmatched); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + it('should apply a delta mutation if gitlab.group_create event received and defaultGroupTransformation in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + await events.publish(mock.group_create_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_group_entity, + removed: [], + }); + }); + + it('should apply a delta mutation if gitlab.group_create event received and user/group transformations in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + function customGroupNameTransformer( + options: GroupNameTransformerOptions, + ): string { + return `${options.group.id}`; + } + + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + groupNameTransformer: customGroupNameTransformer, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_create_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_transformed_group_entity, + removed: [], + }); + }); + + it('should apply a delta mutation if gitlab.group_rename event received and defaultGroupTransformation in place', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.group_rename_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_added_group_entity, + removed: mock.expected_removed_group_entity, + }); + }); + + it('should apply a delta mutation if gitlab.user_create event received', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_create_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_single_user_entity, + removed: [], + }); + }); + + it('should apply a delta mutation if gitlab.user_destroy event received', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_destroy_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: mock.expected_single_user_removed_entity, + }); + }); + + it('should apply a delta mutation if gitlab.user_add_to_group event received and defaultGroupTransformer', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_add_to_group_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_group_user_entity, + removed: [], + }); + }); + + it('should ignore a delta mutation if gitlab.user_add_to_group event received and group outside scope', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_add_to_group_event_mismatched); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should apply a delta mutation if gitlab.user_remove_from_group event received and customGroupTransformer', async () => { + const config = new ConfigReader(mock.config_org_integration); + const schedule = new PersistingTaskRunner(); + const events = DefaultEventsService.create({ logger }); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + function customGroupNameTransformer( + options: GroupNameTransformerOptions, + ): string { + return `${options.group.id}`; + } + + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + groupNameTransformer: customGroupNameTransformer, + logger, + schedule, + events, + })[0]; + + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + await provider.connect(entityProviderConnection); + + await events.publish(mock.user_remove_from_group_event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: mock.expected_transformed_group_entity, + removed: [], }); }); - - it('fail without schedule and scheduler', () => { - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - }, - }, - }, - }); - - expect(() => - GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - }), - ).toThrow('Either schedule or scheduler must be provided'); - }); - - it('fail with scheduler but no schedule config', () => { - const scheduler = { - createScheduledTaskRunner: (_: any) => jest.fn(), - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - }, - }, - }, - }, - }); - - expect(() => - GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ).toThrow( - 'No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:test-id', - ); - }); - - it('fail with scheduler but no group config when host is gitlab.com', () => { - const scheduler = { - createScheduledTaskRunner: (_: any) => jest.fn(), - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'gitlab.com', - apiBaseUrl: 'https://gitlab.com/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'gitlab.com', - orgEnabled: true, - }, - }, - }, - }, - }); - - expect(() => - GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ).toThrow( - `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`, - ); - }); - - it('single simple provider config with schedule in config', async () => { - const schedule = new PersistingTaskRunner(); - const scheduler = { - createScheduledTaskRunner: (_: any) => schedule, - } as unknown as PluginTaskScheduler; - const config = new ConfigReader({ - integrations: { - gitlab: [ - { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', - token: '1234', - }, - ], - }, - catalog: { - providers: { - gitlab: { - 'test-id': { - host: 'test-gitlab', - group: 'test-group', - orgEnabled: true, - schedule: { - frequency: 'PT30M', - timeout: 'PT3M', - }, - }, - }, - }, - }, - }); - const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - logger, - scheduler, - }); - - expect(providers).toHaveLength(1); - expect(providers[0].getProviderName()).toEqual( - 'GitlabOrgDiscoveryEntityProvider:test-id', - ); - }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 90a7a14b6b..aba888aea1 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -25,6 +25,7 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; +import { EventsService } from '@backstage/plugin-events-node'; import { merge } from 'lodash'; import * as uuid from 'uuid'; import { LoggerService } from '@backstage/backend-plugin-api'; @@ -35,21 +36,24 @@ import { paginated, readGitlabConfigs, } from '../lib'; -import { - GitLabGroup, - GitLabUser, - GroupNameTransformer, - GroupTransformer as GroupEntitiesTransformer, - PagedResponse, - UserTransformer, -} from '../lib/types'; import { defaultGroupEntitiesTransformer, defaultGroupNameTransformer, defaultUserTransformer, } from '../lib/defaultTransformers'; +import { + GitLabGroup, + GitLabUser, + GroupTransformer as GroupEntitiesTransformer, + GroupNameTransformer, + PagedResponse, + SystemHookBaseGroupEventsSchema, + SystemHookBaseMembershipEventsSchema, + SystemHookBaseUserEventsSchema, + UserTransformer, +} from '../lib/types'; -type Result = { +type UserResult = { scanned: number; matches: GitLabUser[]; }; @@ -59,6 +63,36 @@ type GroupResult = { matches: GitLabGroup[]; }; +type SystemHookGroupCreateOrDestroyEventSchema = + SystemHookBaseGroupEventsSchema & { + event_name: 'group_create' | 'group_destroy'; + }; + +type SystemHookGroupRenameEventSchema = SystemHookBaseGroupEventsSchema & { + event_name: 'group_rename'; + old_path: string; + old_full_path: string; +}; + +type SystemHookUserCreateOrDestroyEventSchema = + SystemHookBaseUserEventsSchema & { + event_name: 'user_create' | 'user_destroy'; + }; + +type SystemHookCreateOrDestroyMembershipEventsSchema = + SystemHookBaseMembershipEventsSchema & { + event_name: 'user_add_to_group' | 'user_remove_from_group'; + }; + +// System level events +const TOPIC_GROUP_CREATE = 'gitlab.group_create'; +const TOPIC_GROUP_DESTROY = 'gitlab.group_destroy'; +const TOPIC_GROUP_RENAME = 'gitlab.group_rename'; +const TOPIC_USER_CREATE = 'gitlab.user_create'; +const TOPIC_USER_DESTROY = 'gitlab.user_destroy'; +const TOPIC_USER_ADD_GROUP = 'gitlab.user_add_to_group'; +const TOPIC_USER_REMOVE_GROUP = 'gitlab.user_remove_from_group'; + /** * Discovers users and groups from a Gitlab instance. * @public @@ -67,6 +101,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private readonly config: GitlabProviderConfig; private readonly integration: GitLabIntegration; private readonly logger: LoggerService; + private readonly events?: EventsService; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; private userTransformer: UserTransformer; @@ -77,6 +112,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { config: Config, options: { logger: LoggerService; + events?: EventsService; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; userTransformer?: UserTransformer; @@ -96,7 +132,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const integration = integrations.byHost(providerConfig.host); if (!providerConfig.orgEnabled) { - return; + throw new Error(`Org not enabled for ${providerConfig.id}.`); } if (!integration) { @@ -137,6 +173,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { config: GitlabProviderConfig; integration: GitLabIntegration; logger: LoggerService; + events?: EventsService; taskRunner: TaskRunner; userTransformer?: UserTransformer; groupEntitiesTransformer?: GroupEntitiesTransformer; @@ -148,6 +185,8 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { target: this.getProviderName(), }); this.scheduleFn = this.createScheduleFn(options.taskRunner); + this.events = options.events; + this.userTransformer = options.userTransformer ?? defaultUserTransformer; this.groupEntitiesTransformer = options.groupEntitiesTransformer ?? defaultGroupEntitiesTransformer; @@ -162,6 +201,117 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { async connect(connection: EntityProviderConnection): Promise { this.connection = connection; await this.scheduleFn(); + + // Specifies which topics will be listened to. + // The topics from the original GitLab events contain only the string 'gitlab'. These are caught by the GitlabEventRouter Module, who republishes them with a more specific topic 'gitlab.' + if (this.events) { + await this.events.subscribe({ + id: this.getProviderName(), + topics: [ + TOPIC_GROUP_CREATE, + TOPIC_GROUP_DESTROY, + TOPIC_GROUP_RENAME, + TOPIC_USER_CREATE, + TOPIC_USER_DESTROY, + TOPIC_USER_ADD_GROUP, + TOPIC_USER_REMOVE_GROUP, + ], + onEvent: async params => { + this.logger.info(`Received event from topic ${params.topic}`); + + const addEntitiesOperation = (entities: Entity[]) => ({ + removed: [], + added: entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), + })), + }); + + const removeEntitiesOperation = (entities: Entity[]) => ({ + added: [], + removed: entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), + })), + }); + + const replaceEntitiesOperation = (entities: Entity[]) => { + const entitiesToReplace = entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations( + this.integration.config.host, + this.integration.config.baseUrl, + entity, + ), + })); + + return { + removed: entitiesToReplace, + added: entitiesToReplace, + }; + }; + + // handle group change events + if ( + params.topic === TOPIC_GROUP_CREATE || + params.topic === TOPIC_GROUP_DESTROY + ) { + const payload: SystemHookGroupCreateOrDestroyEventSchema = + params.eventPayload as SystemHookGroupCreateOrDestroyEventSchema; + + const createDeltaOperation = + params.topic === TOPIC_GROUP_CREATE + ? addEntitiesOperation + : removeEntitiesOperation; + + await this.onGroupChange(payload, createDeltaOperation); + } + if (params.topic === TOPIC_GROUP_RENAME) { + const payload: SystemHookGroupRenameEventSchema = + params.eventPayload as SystemHookGroupRenameEventSchema; + + await this.onGroupEdit(payload, replaceEntitiesOperation); + } + + // handle user change events + if ( + params.topic === TOPIC_USER_CREATE || + params.topic === TOPIC_USER_DESTROY + ) { + const payload: SystemHookUserCreateOrDestroyEventSchema = + params.eventPayload as SystemHookUserCreateOrDestroyEventSchema; + + const createDeltaOperation = + params.topic === TOPIC_USER_CREATE + ? addEntitiesOperation + : removeEntitiesOperation; + + await this.onUserChange(payload, createDeltaOperation); + } + + // handle user membership changes + if ( + params.topic === TOPIC_USER_ADD_GROUP || + params.topic === TOPIC_USER_REMOVE_GROUP + ) { + const payload: SystemHookCreateOrDestroyMembershipEventsSchema = + params.eventPayload as SystemHookCreateOrDestroyMembershipEventsSchema; + + const createDeltaOperation = addEntitiesOperation; + + await this.onMembershipChange(payload, createDeltaOperation); + } + }, + }); + } } private createScheduleFn(taskRunner: TaskRunner): () => Promise { @@ -208,6 +358,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groups = paginated(options => client.listGroups(options), { page: 1, per_page: 100, + all_available: true, }); users = paginated(options => client.listUsers(options), { @@ -229,7 +380,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const idMappedUser: { [userId: number]: GitLabUser } = {}; - const res: Result = { + const userRes: UserResult = { scanned: 0, matches: [], }; @@ -240,46 +391,42 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }; for await (const user of users) { - if (!this.config.userPattern.test(user.email ?? user.username ?? '')) { - continue; - } + userRes.scanned++; - res.scanned++; - - if (user.state !== 'active') { + if (!this.shouldProcessUser(user)) { + logger.debug(`Skipped user: ${user.username}`); continue; } idMappedUser[user.id] = user; - res.matches.push(user); + userRes.matches.push(user); } for await (const group of groups) { - if (!this.config.groupPattern.test(group.full_path ?? '')) { - continue; - } - - if ( - this.config.group && - !group.full_path.startsWith(`${this.config.group}/`) - ) { - continue; - } - groupRes.scanned++; + + if (!this.shouldProcessGroup(group)) { + logger.debug(`Skipped group: ${group.full_path}`); + continue; + } + logger.debug(`Processed group: ${group.full_path}`); + groupRes.matches.push(group); let groupUsers: PagedResponse = { items: [] }; try { - groupUsers = await client.getGroupMembers(group.full_path, ['DIRECT']); + const relations = this.config.allowInherited + ? ['DIRECT', 'INHERITED'] + : ['DIRECT']; + groupUsers = await client.getGroupMembers(group.full_path, relations); } catch (e) { logger.error( `Failed fetching users for group '${group.full_path}': ${e}`, ); } - for (const groupUser of groupUsers.items) { const user = idMappedUser[groupUser.id]; + if (user) { user.groups = (user.groups ?? []).concat(group); } @@ -288,13 +435,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const groupsWithUsers = groupRes.matches.filter(group => { return ( - res.matches.filter(x => { + userRes.matches.filter(x => { return !!x.groups?.find(y => y.id === group.id); }).length > 0 ); }); - const userEntities = res.matches.map(p => + const userEntities = userRes.matches.map(p => this.userTransformer({ user: p, integrationConfig: this.integration.config, @@ -309,6 +456,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupNameTransformer: this.groupNameTransformer, }); + logger.info( + `Scanned ${userRes.scanned} users and processed ${userEntities.length} users`, + ); + logger.info( + `Scanned ${groupRes.scanned} groups and processed ${groupEntities.length} groups`, + ); + await this.connection.applyMutation({ type: 'full', entities: [...userEntities, ...groupEntities].map(entity => ({ @@ -321,6 +475,301 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { })), }); } + private async onGroupChange( + event: SystemHookGroupCreateOrDestroyEventSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const getClient = () => + new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + let group: GitLabGroup | undefined; + if (event.event_name === 'group_destroy') { + group = { + id: event.group_id, + full_path: event.full_path, + name: event.name, + description: '', + parent_id: 0, + }; + } else { + const client = getClient(); + group = await client.getGroupById(event.group_id); + } + + if (!this.shouldProcessGroup(group)) { + this.logger.debug(`Skipped group ${group.full_path}.`); + return; + } + + // create the group entity + const groupEntity = this.groupEntitiesTransformer({ + groups: [group], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + // we need to fetch the parent group's object because its representation might be changed by the groupTransformer + if (group.parent_id) { + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + const parentGroup = await client.getGroupById(group.parent_id); + + groupEntity[0].spec.parent = this.groupNameTransformer({ + group: parentGroup, + providerConfig: this.config, + }); + } + + this.logger.debug(`Applying mutation for group ${group.full_path}.`); + await this.connection.applyMutation({ + type: 'delta', + ...createDeltaOperation(groupEntity), + }); + } + + // the goal here is to trigger a mutation to remove the old entity and add the new one. + private async onGroupEdit( + event: SystemHookGroupRenameEventSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const groupToRemove: GitLabGroup = { + id: event.group_id, + full_path: event.old_full_path, + name: event.name, + description: '', + parent_id: 0, + }; + + if (!this.shouldProcessGroup(groupToRemove)) { + this.logger.debug(`Skipped group ${groupToRemove.full_path}.`); + return; + } + + const groupEntityToRemove = await this.groupEntitiesTransformer({ + groups: [groupToRemove], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + const groupToAdd = await client.getGroupById(event.group_id); + + if (!this.shouldProcessGroup(groupToAdd)) { + this.logger.debug(`Skipped group ${groupToAdd.full_path}.`); + return; + } + + const groupEntityToAdd = await this.groupEntitiesTransformer({ + groups: [groupToAdd], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + if (groupToAdd.parent_id) { + const parentGroup = await client.getGroupById(groupToAdd.parent_id); + + groupEntityToAdd[0].spec.parent = this.groupNameTransformer({ + group: parentGroup, + providerConfig: this.config, + }); + } + + const { added } = createDeltaOperation([...groupEntityToAdd]); + const { removed } = createDeltaOperation([...groupEntityToRemove]); + + this.logger.debug(`Applying mutation for group ${groupToAdd.full_path}.`); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onUserChange( + event: SystemHookUserCreateOrDestroyEventSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + let user: GitLabUser | undefined; + if (event.event_name === 'user_destroy') { + user = { + id: event.user_id, + username: event.username, + email: event.email, + name: event.name, + state: 'active', // in the delete case it doesn't really matter if the user is active or not + web_url: '', + avatar_url: '', + }; + } else { + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + user = await client.getUserById(event.user_id); + } + + if (!this.shouldProcessUser(user)) { + this.logger.debug(`Skipped user ${user.username}.`); + return; + } + + const userEntity = await this.userTransformer({ + user: user, + integrationConfig: this.integration.config, + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + const { added, removed } = createDeltaOperation([userEntity]); + + this.logger.debug(`Applying mutation for user ${user.username}.`); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + // the goal here is to reconstruct the group either from which the user was removed or to which the user was added. Specifically, we add/remove the new user to/from the spec.member property array of the group entity. The Processor should take care of updating the relations + private async onMembershipChange( + event: SystemHookCreateOrDestroyMembershipEventsSchema, + createDeltaOperation: (entities: Entity[]) => { + added: any[]; + removed: any[]; + }, + ): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const client = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); + + const groupToRebuild: GitLabGroup = await client.getGroupById( + event.group_id, + ); + + // If the group is outside the scope there is no point creating anything related to it. + if (!this.shouldProcessGroup(groupToRebuild)) { + this.logger.debug(`Skipped group ${groupToRebuild.full_path}.`); + return; + } + + const relations = this.config.allowInherited + ? ['DIRECT', 'INHERITED'] + : ['DIRECT']; + + const groupMembers = await client.getGroupMembers( + groupToRebuild.full_path, + relations, + ); + + // if (groupMembers.items.length !== 0) { + // groupMembers.items.forEach(element => { + // if ( + // event.event_name === 'user_remove_from_group' && + // element.username === event.user_username + // ) { + // return; + // } + + // usersToBeAdded.push(element.username); + // }); + // } + + // new members of the group + const usersToBeAdded = + event.event_name === 'user_remove_from_group' + ? groupMembers.items.filter(e => e.username !== event.user_username) + : groupMembers.items; + + const groupEntityToModify = await this.groupEntitiesTransformer({ + groups: [groupToRebuild], + providerConfig: this.config, + groupNameTransformer: this.groupNameTransformer, + }); + + // we need to fetch the parent group's object because its representation might be changed by the groupTransformer + if (groupToRebuild.parent_id) { + const parentGroup = await client.getGroupById(groupToRebuild.parent_id); + + groupEntityToModify[0].spec.parent = this.groupNameTransformer({ + group: parentGroup, + providerConfig: this.config, + }); + } + + if (usersToBeAdded.length !== 0) { + groupEntityToModify[0].spec.members = usersToBeAdded.map(e => e.username); + } + + const { added, removed } = createDeltaOperation([...groupEntityToModify]); + + this.logger.debug( + `Applying mutation for group ${groupToRebuild.full_path}.`, + ); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private shouldProcessGroup(group: GitLabGroup): boolean { + return ( + this.config.groupPattern.test(group.full_path) && + (!this.config.group || + group.full_path.startsWith(`${this.config.group}/`)) + ); + } + + private shouldProcessUser(user: GitLabUser): boolean { + return ( + this.config.userPattern.test(user.email ?? user.username ?? '') && + user.state === 'active' + ); + } private withLocations(host: string, baseUrl: string, entity: Entity): Entity { const location = diff --git a/yarn.lock b/yarn.lock index ad5d48bd8f..ce5c906b86 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5569,7 +5569,24 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab": +"@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend-test-utils": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + luxon: ^3.0.0 + languageName: unknown + linkType: soft + +"@backstage/plugin-catalog-backend-module-gitlab@workspace:^, @backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab" dependencies: @@ -5581,7 +5598,11 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend-test-utils": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + "@gitbeaker/rest": ^39.25.0 "@types/lodash": ^4.14.151 "@types/uuid": ^9.0.0 lodash: ^4.17.21 From 4d87f98ec7fe7bb8c5474863f8ce1915a92514b9 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Sat, 16 Mar 2024 13:29:38 +0100 Subject: [PATCH 02/15] fix: fix branch Signed-off-by: ElaineDeMattosSilvaB --- plugins/catalog-backend-module-gitlab/api-report.md | 2 ++ plugins/catalog-backend-module-gitlab/package.json | 6 +----- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 2 ++ .../src/lib/defaultTransformers.ts | 4 ++++ plugins/catalog-backend-module-gitlab/src/lib/types.ts | 2 ++ yarn.lock | 4 ---- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index 809ac168f2..772df7d37b 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -35,6 +35,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { // (undocumented) getProviderName(): string; // (undocumented) + // (undocumented) refresh(logger: LoggerService): Promise; } @@ -65,6 +66,7 @@ export type GitLabGroup = { name: string; full_path: string; description?: string; + visibility?: string; parent_id?: number; }; diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 54ecf67503..d81db51117 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -15,7 +15,7 @@ */ { "name": "@backstage/plugin-catalog-backend-module-gitlab", - "version": "0.3.10-next.0", + "version": "0.3.11-next.2", "description": "A Backstage catalog backend module that helps integrate towards GitLab", "version": "0.3.15", "main": "src/index.ts", @@ -71,10 +71,7 @@ "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/integration": "workspace:^", - "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", - "@backstage/plugin-events-node": "workspace:^", - "@gitbeaker/rest": "^39.25.0", "lodash": "^4.17.21", "node-fetch": "^2.6.7", "uuid": "^9.0.0" @@ -82,7 +79,6 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", - "@backstage/plugin-events-backend-test-utils": "workspace:^", "@types/lodash": "^4.14.151", "@types/uuid": "^9.0.0", "luxon": "^3.0.0", diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index b30da0461a..8f56657f68 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -173,6 +173,7 @@ export class GitLabClient { name description fullPath + visibility parent { id } @@ -207,6 +208,7 @@ export class GitLabClient { name: groupItem.name, description: groupItem.description, full_path: groupItem.fullPath, + visibility: groupItem.visibility, parent_id: Number( groupItem.parent.id.replace(/^gid:\/\/gitlab\/Group\//, ''), ), diff --git a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts index d7a6bf76e9..912e5a0502 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts @@ -49,6 +49,10 @@ export function defaultGroupEntitiesTransformer( const annotations: { [annotationName: string]: string } = {}; annotations[`${options.providerConfig.host}/team-path`] = group.full_path; + if (group.visibility !== undefined) { + annotations[`${options.providerConfig.host}/visibility`] = + group.visibility; + } const entity: GroupEntity = { apiVersion: 'backstage.io/v1alpha1', diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index b60814ade1..2857fea162 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -79,6 +79,7 @@ export type GitLabGroup = { name: string; full_path: string; description?: string; + visibility?: string; parent_id?: number; }; @@ -118,6 +119,7 @@ export type GitLabDescendantGroupsResponse = { name: string; description: string; fullPath: string; + visibility: string; parent: { id: string; }; diff --git a/yarn.lock b/yarn.lock index ce5c906b86..78162415e2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5598,11 +5598,7 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" - "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" - "@backstage/plugin-events-backend-test-utils": "workspace:^" - "@backstage/plugin-events-node": "workspace:^" - "@gitbeaker/rest": ^39.25.0 "@types/lodash": ^4.14.151 "@types/uuid": ^9.0.0 lodash: ^4.17.21 From 36f00812797ab0b412466740013b82d3887de9d8 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Sat, 16 Mar 2024 13:41:48 +0100 Subject: [PATCH 03/15] fix: fix imports Signed-off-by: ElaineDeMattosSilvaB --- .../catalog-backend-module-gitlab/package.json | 4 ++++ .../providers/GitlabOrgDiscoveryEntityProvider.ts | 15 +-------------- yarn.lock | 6 +++++- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index d81db51117..302126e015 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -71,7 +71,10 @@ "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/integration": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", + "@gitbeaker/rest": "^39.34.3", "lodash": "^4.17.21", "node-fetch": "^2.6.7", "uuid": "^9.0.0" @@ -79,6 +82,7 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", + "@backstage/plugin-events-backend-test-utils": "workspace:^", "@types/lodash": "^4.14.151", "@types/uuid": "^9.0.0", "luxon": "^3.0.0", diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index aba888aea1..30105dcc1f 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -626,7 +626,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ); } - let user: GitLabUser | undefined; + let user: GitLabUser; if (event.event_name === 'user_destroy') { user = { id: event.user_id, @@ -705,19 +705,6 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { relations, ); - // if (groupMembers.items.length !== 0) { - // groupMembers.items.forEach(element => { - // if ( - // event.event_name === 'user_remove_from_group' && - // element.username === event.user_username - // ) { - // return; - // } - - // usersToBeAdded.push(element.username); - // }); - // } - // new members of the group const usersToBeAdded = event.event_name === 'user_remove_from_group' diff --git a/yarn.lock b/yarn.lock index 78162415e2..2e56066685 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5598,7 +5598,11 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend-test-utils": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + "@gitbeaker/rest": ^39.34.3 "@types/lodash": ^4.14.151 "@types/uuid": ^9.0.0 lodash: ^4.17.21 @@ -11348,7 +11352,7 @@ __metadata: languageName: node linkType: hard -"@gitbeaker/rest@npm:^39.25.0": +"@gitbeaker/rest@npm:^39.25.0, @gitbeaker/rest@npm:^39.34.3": version: 39.34.3 resolution: "@gitbeaker/rest@npm:39.34.3" dependencies: From bb3fbd083b93c3282e475727e98fd70ecc6b119a Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Sat, 16 Mar 2024 18:20:36 +0100 Subject: [PATCH 04/15] fix: add api-report Signed-off-by: ElaineDeMattosSilvaB --- plugins/catalog-backend-module-gitlab/api-report.md | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index 772df7d37b..80a9d0b8b7 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -35,7 +35,6 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { // (undocumented) getProviderName(): string; // (undocumented) - // (undocumented) refresh(logger: LoggerService): Promise; } From aef7d0e269e2eb33645a75e51ebf8e4ac952bcee Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Thu, 18 Apr 2024 08:56:40 +0200 Subject: [PATCH 05/15] fix: merge conflicts Signed-off-by: ElaineDeMattosSilvaB --- .../package.json | 118 +++++++----------- yarn.lock | 6 +- 2 files changed, 48 insertions(+), 76 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 302126e015..5490ac80bc 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -1,45 +1,58 @@ -/* - * Copyright 2024 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ { - "name": "@backstage/plugin-catalog-backend-module-gitlab", - "version": "0.3.11-next.2", + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/integration": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7", + "uuid": "^9.0.0" + }, "description": "A Backstage catalog backend module that helps integrate towards GitLab", - "version": "0.3.15", - "main": "src/index.ts", - "types": "src/index.ts", - "license": "Apache-2.0", - "publishConfig": { - "access": "public" + "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@types/lodash": "^4.14.151", + "@types/uuid": "^9.0.0", + "luxon": "^3.0.0", + "msw": "^1.0.0" }, - "keywords": [ - "backstage" - ], - "homepage": "https://backstage.io", - "repository": { - "type": "git", - "url": "https://github.com/backstage/backstage", - "directory": "plugins/catalog-backend-module-gitlab" - }, - "license": "Apache-2.0", "exports": { ".": "./src/index.ts", "./alpha": "./src/alpha.ts", "./package.json": "./package.json" }, + "files": [ + "config.d.ts", + "dist" + ], + "homepage": "https://backstage.io", + "keywords": [ + "backstage" + ], + "license": "Apache-2.0", "main": "src/index.ts", + "name": "@backstage/plugin-catalog-backend-module-gitlab", + "publishConfig": { + "access": "public" + }, + "repository": { + "directory": "plugins/catalog-backend-module-gitlab", + "type": "git", + "url": "https://github.com/backstage/backstage" + }, + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "postpack": "backstage-cli package postpack", + "prepack": "backstage-cli package prepack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, "types": "src/index.ts", "typesVersions": { "*": { @@ -51,42 +64,5 @@ ] } }, - "files": [ - "config.d.ts", - "dist" - ], - "scripts": { - "build": "backstage-cli package build", - "clean": "backstage-cli package clean", - "lint": "backstage-cli package lint", - "prepack": "backstage-cli package prepack", - "postpack": "backstage-cli package postpack", - "start": "backstage-cli package start", - "test": "backstage-cli package test" - }, - "dependencies": { - "@backstage/backend-common": "workspace:^", - "@backstage/backend-plugin-api": "workspace:^", - "@backstage/backend-tasks": "workspace:^", - "@backstage/catalog-model": "workspace:^", - "@backstage/config": "workspace:^", - "@backstage/integration": "workspace:^", - "@backstage/plugin-catalog-common": "workspace:^", - "@backstage/plugin-catalog-node": "workspace:^", - "@backstage/plugin-events-node": "workspace:^", - "@gitbeaker/rest": "^39.34.3", - "lodash": "^4.17.21", - "node-fetch": "^2.6.7", - "uuid": "^9.0.0" - }, - "devDependencies": { - "@backstage/backend-test-utils": "workspace:^", - "@backstage/cli": "workspace:^", - "@backstage/plugin-events-backend-test-utils": "workspace:^", - "@types/lodash": "^4.14.151", - "@types/uuid": "^9.0.0", - "luxon": "^3.0.0", - "msw": "^1.0.0" - }, - "configSchema": "config.d.ts" + "version": "0.3.14" } diff --git a/yarn.lock b/yarn.lock index 2e56066685..78162415e2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5598,11 +5598,7 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" - "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" - "@backstage/plugin-events-backend-test-utils": "workspace:^" - "@backstage/plugin-events-node": "workspace:^" - "@gitbeaker/rest": ^39.34.3 "@types/lodash": ^4.14.151 "@types/uuid": ^9.0.0 lodash: ^4.17.21 @@ -11352,7 +11348,7 @@ __metadata: languageName: node linkType: hard -"@gitbeaker/rest@npm:^39.25.0, @gitbeaker/rest@npm:^39.34.3": +"@gitbeaker/rest@npm:^39.25.0": version: 39.34.3 resolution: "@gitbeaker/rest@npm:39.34.3" dependencies: From c3c2591abf2addb062cf26ce02c805510b7f9e03 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Wed, 20 Mar 2024 22:26:18 +0100 Subject: [PATCH 06/15] chore: add documentation and fix package.json Signed-off-by: ElaineDeMattosSilvaB --- docs/integrations/gitlab/discovery.md | 155 +++++++++++++++++++------ docs/integrations/gitlab/org.md | 159 +++++++++++++++++++++++++- 2 files changed, 276 insertions(+), 38 deletions(-) diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index d41f4f8de5..668b0de694 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -11,6 +11,116 @@ entities from GitLab. The entity provider will crawl the GitLab instance and reg entities matching the configured paths. This can be useful as an alternative to static locations or manually adding things to the catalog. +This provider can also be configured to ingest GitLab data based on [GitLab Webhooks](https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#configure-a-webhook-in-gitlab). The events currently accepted are: + +- [`push`](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#push-events). + +## Installation + +As this provider is not one of the default providers, you will first need to install +the gitlab catalog plugin: + +```bash +# From your Backstage root directory +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab +``` + +### Installation with New Backend System + +Then add the following to your backend initialization: + +```ts title="packages/backend/src/index.ts +// optional if you want HTTP endpoints to receive external events +// backend.add(import('@backstage/plugin-events-backend/alpha')); +// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events +// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha')); +// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md +// backend.add(eventsModuleGitlabEventRouter()); +// optional - token validator for the gitlab topic +// backend.add(eventsModuleGitlabWebhook()); +backend.add(import('@backstage/plugin-catalog-backend-module-gitlab/alpha')); +``` + +You need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Further documentation: + +- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md) + +### Installation with Legacy Backend System + +#### Installation without Events Support + +Add the segment below to `packages/backend/src/plugins/catalog.ts`: + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + /* highlight-add-start */ + builder.addEntityProvider( + GitlabDiscoveryEntityProvider.fromConfig(env.config, { + logger: env.logger, + scheduler: env.scheduler, + }), + ); + /* highlight-add-end */ + + // .. +} +``` + +#### Installation with Events Support + +Please follow the installation instructions at + +- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md) + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```ts title="packages/backend/src/plugins/catalog.ts" +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; +/* highlight-add-next-line */ +import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; +import { Router } from 'express'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + builder.addProcessor(new ScaffolderEntitiesProcessor()); + /* highlight-add-start */ + const gitlabProvider = GitlabDiscoveryEntityProvider.fromConfig(env.config, { + logger: env.logger, + scheduler: env.scheduler, + }); + env.eventBroker.subscribe(gitlabProvider); + builder.addEntityProvider(gitlabProvider); + /* highlight-add-end */ + const { processingEngine, router } = await builder.build(); + await processingEngine.start(); + return router; +} +``` + +## Configuration + To use the discovery provider, you'll need a GitLab integration [set up](locations.md) with a `token`. Then you can add a provider config per group to the catalog configuration: @@ -34,44 +144,8 @@ catalog: timeout: { minutes: 3 } ``` -As this provider is not one of the default providers, you will first need to install -the gitlab catalog plugin: - -```bash -# From your Backstage root directory -yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab -``` - -Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`: - -```ts title="packages/backend/src/plugins/catalog.ts" -import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; - -const builder = await CatalogBuilder.create(env); -/** ... other processors and/or providers ... */ -builder.addEntityProvider( - ...GitlabDiscoveryEntityProvider.fromConfig(env.config, { - logger: env.logger, - // optional: alternatively, use scheduler with schedule defined in app-config.yaml - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 30 }, - timeout: { minutes: 3 }, - }), - // optional: alternatively, use schedule - scheduler: env.scheduler, - }), -); -``` - ## Alternative processor -```yaml -catalog: - locations: - - type: gitlab-discovery - target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml -``` - As alternative to the entity provider `GitlabDiscoveryEntityProvider` you can still use the `GitLabDiscoveryProcessor`. @@ -109,6 +183,15 @@ export default async function createPlugin( } ``` +And add the following to your app-config.yaml + +```yaml +catalog: + locations: + - type: gitlab-discovery + target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml +``` + If you don't want create location object if file with component definition do not exists in project, you can set the `skipReposWithoutExactFileMatch` option. That can reduce count of request to gitlab with 404 status code. If you don't want to create location object if the project is a fork, you can set the `skipForkedRepos` option. diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index f2a483c9d1..c748fba1ba 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -11,14 +11,57 @@ groups -- directly from GitLab. The result is a hierarchy of [`Group`](../../features/software-catalog/descriptor-format.md#kind-group) entities that mirrors your org setup. +This provider can also be configured to ingest GitLab data based on [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html). The events currently accepted are: + +- `group_create` +- `group_destroy` +- `group_rename` +- `user_create` +- `user_destroy` +- `user_add_to_group` +- `user_remove_from_group` + +## Installation + As this provider is not one of the default providers, you will first need to install the Gitlab provider plugin: ```bash # From your Backstage root directory -yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-gitlab @backstage/plugin-catalog-backend-module-gitlab-org ``` -Then add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`: +### Installation with New Backend System + +Then add the following to your backend initialization: + +```ts title="packages/backend/src/index.ts +// optional if you want HTTP endpoints to receive external events +// backend.add(import('@backstage/plugin-events-backend/alpha')); +// optional if you want to use AWS SQS instead of HTTP endpoints to receive external events +// backend.add(import('@backstage/plugin-events-backend-module-aws-sqs/alpha')); +// optional - event router for gitlab. See.: https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md +// backend.add(eventsModuleGitlabEventRouter()); +// optional - token validator for the gitlab topic +// backend.add(eventsModuleGitlabWebhook()); +backend.add(import('@backstage/plugin-catalog-backend-module-gitlab-org')); +``` + +You need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/blob/master/plugins/events-backend/README.md#configuration) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Further documentation: + +- [GitLab System hooks](https://docs.gitlab.com/ee/administration/system_hooks.html) +- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md) + +### Installation with Legacy Backend System + +#### Installation without Events Support + +Add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`: ```ts /* packages/backend/src/plugins/catalog.ts */ @@ -41,6 +84,50 @@ builder.addEntityProvider( /* highlight-add-end */ ``` +#### Installation with Events Support + +Please follow the installation instructions at + +- [Events Plugin](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [GitLab Module for the Events Plugin](https://github.com/backstage/backstage/blob/master/plugins/events-backend-module-gitlab/README.md) + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```ts title="packages/backend/src/plugins/catalog.ts" +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; +/* highlight-add-next-line */ +import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; +import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; +import { Router } from 'express'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + builder.addProcessor(new ScaffolderEntitiesProcessor()); + /* highlight-add-start */ + const gitlabOrgProvider = GitlabOrgDiscoveryEntityProvider.fromConfig( + env.config, + { + logger: env.logger, + scheduler: env.scheduler, + }, + ); + env.eventBroker.subscribe(gitlabOrgProvider); + builder.addEntityProvider(gitlabOrgProvider); + /* highlight-add-end */ + const { processingEngine, router } = await builder.build(); + await processingEngine.start(); + return router; +} +``` + ## Configuration To use the entity provider, you'll need a [Gitlab integration set up](https://backstage.io/docs/integrations/gitlab/locations). @@ -107,6 +194,74 @@ catalog: - allow: [Group, User] ``` +### Custom Transformers + +You can inject your own transformation logic to help map GitLab API responses +into Backstage entities. You can do this on the user and group requests to +enable you to do further processing or updates to these entities. + +To enable this you pass a function into the `GitlabOrgDiscoveryEntityProvider`. You can +pass a `UserTransformer`, a `GroupEntitiesTransformer` or a `GroupNameTransformer` (or all of them)). The function is invoked +for each item (user or group) that is returned from the API. + +The example below uses the groupNameTransformer option to change the metadata.name property of the Backstage Group Entity. Instead of populating it with the usual `group.full_path` data that comes from GitLab, it uses the `group.id`: + +```ts +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; +import { + GitlabOrgDiscoveryEntityProvider, + /* highlight-add-next-line */ + GroupNameTransformerOptions, +} from '@backstage/plugin-catalog-backend-module-gitlab'; + +/* highlight-add-start */ +function customGroupNameTransformer( + options: GroupNameTransformerOptions, +): string { + return `${options.group.id}`; +} +/* highlight-add-end */ + +/** + * Registers the GitlabDiscoveryEntityProvider with the catalog processing extension point. + * + * @alpha + */ +export const catalogModuleGitlabOrgDiscoveryEntityProvider = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'gitlabOrgDiscoveryEntityProvider', + register(env) { + env.registerInit({ + deps: { + config: coreServices.rootConfig, + catalog: catalogProcessingExtensionPoint, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + events: eventsServiceRef, + }, + async init({ config, catalog, logger, scheduler, events }) { + const gitlabOrgDiscoveryEntityProvider = + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + /* highlight-add-next-line */ + groupNameTransformer: customGroupNameTransformer, + logger: loggerToWinstonLogger(logger), + events, + scheduler, + }); + catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider); + }, + }); + }, + }); +``` + ## Troubleshooting **NOTE**: If any groups that are being ingested are empty groups (i.e. do not From 63336c8710ee605fac84254d5c98339078bd86ed Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Wed, 20 Mar 2024 22:48:56 +0100 Subject: [PATCH 07/15] fix: solve uninstalled deps Signed-off-by: ElaineDeMattosSilvaB --- yarn.lock | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yarn.lock b/yarn.lock index 78162415e2..ce5c906b86 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5598,7 +5598,11 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend-test-utils": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + "@gitbeaker/rest": ^39.25.0 "@types/lodash": ^4.14.151 "@types/uuid": ^9.0.0 lodash: ^4.17.21 From c2f20d5803856a6ddd903094dc689cfe51922002 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Mon, 25 Mar 2024 20:15:34 +0100 Subject: [PATCH 08/15] feat: incorporate review changes Signed-off-by: ElaineDeMattosSilvaB --- .../src/__testUtils__/handlers.ts | 16 ++ .../src/__testUtils__/mocks.ts | 94 +++++++++--- .../GitlabDiscoveryEntityProvider.test.ts | 2 +- .../GitlabDiscoveryEntityProvider.ts | 26 ++-- .../GitlabOrgDiscoveryEntityProvider.test.ts | 4 +- .../GitlabOrgDiscoveryEntityProvider.ts | 141 +++++++++--------- 6 files changed, 176 insertions(+), 107 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts index 2f464dc1fe..00d4994e94 100644 --- a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts @@ -210,6 +210,22 @@ const graphqlHandlers = [ }), ); } + // group with no associated members + if (group === 'group3' && relations.includes('DIRECT')) { + return res( + ctx.data({ + group: { + groupMembers: { + nodes: [{}], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ); + } if (group === 'saas-multi-user-group') { return res( ctx.data({ diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts index 9e0f55d05d..5a078ab689 100644 --- a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts @@ -139,7 +139,7 @@ export const config_saas_no_group: MockObject = { }, }; -export const config_non_gitlab_host: MockObject = { +export const config_github_host: MockObject = { integrations: { github: [ { @@ -910,9 +910,9 @@ export const group_destroy_event: EventParams = { event_name: 'group_destroy', created_at: '2024-02-02T10:53:09Z', updated_at: '2024-02-02T10:53:09Z', - name: 'group1', - path: 'group1', - full_path: 'group1', + name: 'group3', + path: 'group3', + full_path: 'group3', group_id: 123, }, }; @@ -951,10 +951,10 @@ export const group_create_event: EventParams = { event_name: 'group_create', created_at: '2024-02-02T10:53:09Z', updated_at: '2024-02-02T10:53:09Z', - name: 'group1', - path: 'group1', - full_path: 'group1', - group_id: 1, + name: 'group3', + path: 'group3', + full_path: 'group3', + group_id: 3, }, }; export const group_create_event_unmatched: EventParams = { @@ -1038,9 +1038,9 @@ export const user_remove_from_group_event: EventParams = { eventPayload: { created_at: '2024-02-02T10:53:09Z', updated_at: '2024-02-02T10:53:09Z', - group_name: 'group1', - group_path: 'my-groups/group1', - group_id: 1, + group_name: 'group3', + group_path: 'my-groups/group3', + group_id: 3, user_username: 'user1', user_name: 'John Doe', user_email: 'john.doe@company.com', @@ -1382,17 +1382,17 @@ export const expected_group_entity: MockObject[] = [ kind: 'Group', metadata: { annotations: { - 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-location': 'url:https://example.com/group3', 'backstage.io/managed-by-origin-location': - 'url:https://example.com/group1', - 'example.com/team-path': 'group1', + 'url:https://example.com/group3', + 'example.com/team-path': 'group3', }, - name: 'group1', + name: 'group3', }, spec: { children: [], profile: { - displayName: 'group1', + displayName: 'group3', }, type: 'team', }, @@ -1489,17 +1489,44 @@ export const expected_transformed_group_entity: MockObject[] = [ kind: 'Group', metadata: { annotations: { - 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-location': 'url:https://example.com/group3', 'backstage.io/managed-by-origin-location': - 'url:https://example.com/group1', - 'example.com/team-path': 'group1', + 'url:https://example.com/group3', + 'example.com/team-path': 'group3', }, - name: '1', + name: '3', }, spec: { children: [], profile: { - displayName: 'group1', + displayName: 'group3', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, +]; + +export const expected_removed_user_entity: MockObject[] = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group3', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group3', + 'example.com/team-path': 'group3', + }, + name: '3', + }, + spec: { + children: [], + members: [], + profile: { + displayName: 'group3', }, type: 'team', }, @@ -1661,8 +1688,29 @@ export const expected_full_org_scan_entities: MockObject[] = [ }, locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', }, - - expected_group_entity[0], + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:https://example.com/group1', + 'backstage.io/managed-by-origin-location': + 'url:https://example.com/group1', + 'example.com/team-path': 'group1', + }, + name: 'group1', + }, + spec: { + children: [], + profile: { + displayName: 'group1', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, ]; export const expected_full_org_scan_entities_saas: MockObject[] = [ diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index bd725ee0ce..7167fc1c2d 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -87,7 +87,7 @@ describe('GitlabDiscoveryEntityProvider - configuration', () => { }); it('should throw error when no matching GitLab integration config found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader(mock.config_non_gitlab_host); + const config = new ConfigReader(mock.config_github_host); expect(() => { GitlabDiscoveryEntityProvider.fromConfig(config, { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index 653b12060e..73ff5f3a01 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -61,6 +61,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; private readonly events?: EventsService; + private readonly gitLabClient: GitLabClient; static fromConfig( config: Config, @@ -129,6 +130,10 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }); this.scheduleFn = this.createScheduleFn(options.taskRunner); this.events = options.events; + this.gitLabClient = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); } getProviderName(): string { @@ -196,13 +201,9 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { `Gitlab discovery connection not initialized for ${this.getProviderName()}`, ); } - const client = new GitLabClient({ - config: this.integration.config, - logger: logger, - }); const projects = paginated( - options => client.listProjects(options), + options => this.gitLabClient.listProjects(options), { archived: false, group: this.config.group, @@ -217,13 +218,17 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { }; for await (const project of projects) { - if (await this.shouldProcessProject(project, client)) { + if (await this.shouldProcessProject(project, this.gitLabClient)) { res.scanned++; res.matches.push(project); } } const locations = res.matches.map(p => this.createLocationSpec(p)); + + logger.info( + `Processed ${locations.length} from scanned ${res.scanned} projects.`, + ); await this.connection.applyMutation({ type: 'full', entities: locations.map(location => ({ @@ -261,12 +266,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { `Received push event for ${event.project.path_with_namespace}`, ); - const client = new GitLabClient({ - config: this.integration.config, - logger: this.logger, - }); - - const project = await client.getProjectById(event.project_id); + const project = await this.gitLabClient.getProjectById(event.project_id); if (!project) { this.logger.debug( @@ -276,7 +276,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { return; } - if (!(await this.shouldProcessProject(project, client))) { + if (!(await this.shouldProcessProject(project, this.gitLabClient))) { this.logger.debug(`Skipping event ${event.project.path_with_namespace}`); return; } diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index a65b0a1bc2..b5a049538d 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -63,7 +63,7 @@ describe('GitlabOrgDiscoveryEntityProvider - configuration', () => { it('should throw error when no matching GitLab integration config found', () => { const schedule = new PersistingTaskRunner(); - const config = new ConfigReader(mock.config_non_gitlab_host); + const config = new ConfigReader(mock.config_github_host); expect(() => { GitlabOrgDiscoveryEntityProvider.fromConfig(config, { @@ -625,7 +625,7 @@ describe('GitlabOrgDiscoveryEntityProvider with events support', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ type: 'delta', - added: mock.expected_transformed_group_entity, + added: mock.expected_removed_user_entity, removed: [], }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 30105dcc1f..0135c7b4a9 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -107,6 +107,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { private userTransformer: UserTransformer; private groupEntitiesTransformer: GroupEntitiesTransformer; private groupNameTransformer: GroupNameTransformer; + private readonly gitLabClient: GitLabClient; static fromConfig( config: Config, @@ -192,6 +193,11 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { options.groupEntitiesTransformer ?? defaultGroupEntitiesTransformer; this.groupNameTransformer = options.groupNameTransformer ?? defaultGroupNameTransformer; + + this.gitLabClient = new GitLabClient({ + config: this.integration.config, + logger: this.logger, + }); } getProviderName(): string { @@ -346,31 +352,33 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ); } - const client = new GitLabClient({ - config: this.integration.config, - logger: logger, - }); - let groups; let users; - if (client.isSelfManaged()) { - groups = paginated(options => client.listGroups(options), { - page: 1, - per_page: 100, - all_available: true, - }); + if (this.gitLabClient.isSelfManaged()) { + groups = paginated( + options => this.gitLabClient.listGroups(options), + { + page: 1, + per_page: 100, + all_available: true, + }, + ); - users = paginated(options => client.listUsers(options), { - page: 1, - per_page: 100, - active: true, - }); + users = paginated( + options => this.gitLabClient.listUsers(options), + { + page: 1, + per_page: 100, + active: true, + }, + ); } else { - groups = (await client.listDescendantGroups(this.config.group)).items; + groups = (await this.gitLabClient.listDescendantGroups(this.config.group)) + .items; const rootGroup = this.config.group.split('/')[0]; users = paginated( - options => client.listSaaSUsers(rootGroup, options), + options => this.gitLabClient.listSaaSUsers(rootGroup, options), { page: 1, per_page: 100, @@ -418,7 +426,10 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { const relations = this.config.allowInherited ? ['DIRECT', 'INHERITED'] : ['DIRECT']; - groupUsers = await client.getGroupMembers(group.full_path, relations); + groupUsers = await this.gitLabClient.getGroupMembers( + group.full_path, + relations, + ); } catch (e) { logger.error( `Failed fetching users for group '${group.full_path}': ${e}`, @@ -488,12 +499,6 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ); } - const getClient = () => - new GitLabClient({ - config: this.integration.config, - logger: this.logger, - }); - let group: GitLabGroup | undefined; if (event.event_name === 'group_destroy') { group = { @@ -504,8 +509,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { parent_id: 0, }; } else { - const client = getClient(); - group = await client.getGroupById(event.group_id); + group = await this.gitLabClient.getGroupById(event.group_id); } if (!this.shouldProcessGroup(group)) { @@ -522,12 +526,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { // we need to fetch the parent group's object because its representation might be changed by the groupTransformer if (group.parent_id) { - const client = new GitLabClient({ - config: this.integration.config, - logger: this.logger, - }); - - const parentGroup = await client.getGroupById(group.parent_id); + const parentGroup = await this.gitLabClient.getGroupById(group.parent_id); groupEntity[0].spec.parent = this.groupNameTransformer({ group: parentGroup, @@ -575,12 +574,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupNameTransformer: this.groupNameTransformer, }); - const client = new GitLabClient({ - config: this.integration.config, - logger: this.logger, - }); - - const groupToAdd = await client.getGroupById(event.group_id); + const groupToAdd = await this.gitLabClient.getGroupById(event.group_id); if (!this.shouldProcessGroup(groupToAdd)) { this.logger.debug(`Skipped group ${groupToAdd.full_path}.`); @@ -594,7 +588,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); if (groupToAdd.parent_id) { - const parentGroup = await client.getGroupById(groupToAdd.parent_id); + const parentGroup = await this.gitLabClient.getGroupById( + groupToAdd.parent_id, + ); groupEntityToAdd[0].spec.parent = this.groupNameTransformer({ group: parentGroup, @@ -602,8 +598,8 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); } - const { added } = createDeltaOperation([...groupEntityToAdd]); - const { removed } = createDeltaOperation([...groupEntityToRemove]); + const { added } = createDeltaOperation(groupEntityToAdd); + const { removed } = createDeltaOperation(groupEntityToRemove); this.logger.debug(`Applying mutation for group ${groupToAdd.full_path}.`); await this.connection.applyMutation({ @@ -626,7 +622,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ); } - let user: GitLabUser; + let user: GitLabUser | undefined = undefined; + + // if user destroy event is received, retrieve user data from the event itself if (event.event_name === 'user_destroy') { user = { id: event.user_id, @@ -637,13 +635,18 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { web_url: '', avatar_url: '', }; - } else { - const client = new GitLabClient({ - config: this.integration.config, - logger: this.logger, - }); + } - user = await client.getUserById(event.user_id); + // if user create event received fetch data from gitlab + if (event.event_name === 'user_create') { + user = await this.gitLabClient.getUserById(event.user_id); + } + + if (!user) { + this.logger.debug( + `Couldn't retrieve user data. Skipped ${event.event_name} event processing for user ${event.username}`, + ); + return; } if (!this.shouldProcessUser(user)) { @@ -681,16 +684,19 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ); } - const client = new GitLabClient({ - config: this.integration.config, - logger: this.logger, - }); - - const groupToRebuild: GitLabGroup = await client.getGroupById( + // fetch group data from GitLab + const groupToRebuild: GitLabGroup = await this.gitLabClient.getGroupById( event.group_id, ); - // If the group is outside the scope there is no point creating anything related to it. + if (!groupToRebuild) { + this.logger.debug( + `Couldn't retrieve group data. Skipped ${event.event_name} event processing.`, + ); + return; + } + + // if the group is outside the scope there is no point creating anything related to it. if (!this.shouldProcessGroup(groupToRebuild)) { this.logger.debug(`Skipped group ${groupToRebuild.full_path}.`); return; @@ -700,16 +706,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ? ['DIRECT', 'INHERITED'] : ['DIRECT']; - const groupMembers = await client.getGroupMembers( + // fetch group members from GitLab + const groupMembers = await this.gitLabClient.getGroupMembers( groupToRebuild.full_path, relations, ); - - // new members of the group - const usersToBeAdded = - event.event_name === 'user_remove_from_group' - ? groupMembers.items.filter(e => e.username !== event.user_username) - : groupMembers.items; + const usersToBeAdded = groupMembers.items; const groupEntityToModify = await this.groupEntitiesTransformer({ groups: [groupToRebuild], @@ -719,19 +721,22 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { // we need to fetch the parent group's object because its representation might be changed by the groupTransformer if (groupToRebuild.parent_id) { - const parentGroup = await client.getGroupById(groupToRebuild.parent_id); + const parentGroup = await this.gitLabClient.getGroupById( + groupToRebuild.parent_id, + ); + // update parent of the group entity groupEntityToModify[0].spec.parent = this.groupNameTransformer({ group: parentGroup, providerConfig: this.config, }); } - if (usersToBeAdded.length !== 0) { - groupEntityToModify[0].spec.members = usersToBeAdded.map(e => e.username); - } + // update members of the group entity + groupEntityToModify[0].spec.members = + usersToBeAdded.length !== 0 ? usersToBeAdded.map(e => e.username) : []; - const { added, removed } = createDeltaOperation([...groupEntityToModify]); + const { added, removed } = createDeltaOperation(groupEntityToModify); this.logger.debug( `Applying mutation for group ${groupToRebuild.full_path}.`, From d2928ac3c4613444cb2aa8f1a73f1289e82615c3 Mon Sep 17 00:00:00 2001 From: elaine-mattos Date: Fri, 29 Mar 2024 17:13:47 +0100 Subject: [PATCH 09/15] chore: add yarn.lock Signed-off-by: elaine-mattos --- yarn.lock | 4 ---- 1 file changed, 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index ce5c906b86..78162415e2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5598,11 +5598,7 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" - "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" - "@backstage/plugin-events-backend-test-utils": "workspace:^" - "@backstage/plugin-events-node": "workspace:^" - "@gitbeaker/rest": ^39.25.0 "@types/lodash": ^4.14.151 "@types/uuid": ^9.0.0 lodash: ^4.17.21 From 0b208f831d6d4714fbadb19ff5e1fb3078cec5e9 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Sat, 6 Apr 2024 08:20:33 +0200 Subject: [PATCH 10/15] fix: add allowInherited to readGitlabConfig function Signed-off-by: ElaineDeMattosSilvaB --- .../src/providers/config.test.ts | 4 ++++ plugins/catalog-backend-module-gitlab/src/providers/config.ts | 3 +++ 2 files changed, 7 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index bac56cc6f3..09b42e6475 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -58,6 +58,7 @@ describe('config', () => { groupPattern: /[\s\S]*/, userPattern: /[\s\S]*/, orgEnabled: false, + allowInherited: false, schedule: undefined, skipForkedRepos: false, }), @@ -95,6 +96,7 @@ describe('config', () => { groupPattern: /[\s\S]*/, userPattern: /[\s\S]*/, orgEnabled: false, + allowInherited: false, schedule: undefined, skipForkedRepos: false, }), @@ -133,6 +135,7 @@ describe('config', () => { groupPattern: /[\s\S]*/, userPattern: /[\s\S]*/, orgEnabled: false, + allowInherited: false, schedule: undefined, skipForkedRepos: true, }), @@ -173,6 +176,7 @@ describe('config', () => { groupPattern: /[\s\S]*/, userPattern: /[\s\S]*/, orgEnabled: false, + allowInherited: false, skipForkedRepos: false, schedule: { frequency: Duration.fromISO('PT30M'), diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index c34cb81808..8606a525bd 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -43,6 +43,8 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { config.getOptionalString('groupPattern') ?? /[\s\S]*/, ); const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false; + const allowInherited: boolean = + config.getOptionalBoolean('allowInherited') ?? false; const skipForkedRepos: boolean = config.getOptionalBoolean('skipForkedRepos') ?? false; @@ -62,6 +64,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { groupPattern, schedule, orgEnabled, + allowInherited, skipForkedRepos, }; } From 38dfefed0e9b01de655ad3253a1c8d2d0e6e2f83 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Mon, 15 Apr 2024 19:40:57 +0200 Subject: [PATCH 11/15] feat: improve documentation Signed-off-by: ElaineDeMattosSilvaB --- docs/integrations/gitlab/discovery.md | 23 ++++++++++--- docs/integrations/gitlab/org.md | 47 +++++++++++++++++++-------- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 668b0de694..9a588b3796 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -67,13 +67,18 @@ export default async function createPlugin( const builder = await CatalogBuilder.create(env); /* highlight-add-start */ builder.addEntityProvider( - GitlabDiscoveryEntityProvider.fromConfig(env.config, { + ...GitlabDiscoveryEntityProvider.fromConfig(env.config, { logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // optional: alternatively, use schedule scheduler: env.scheduler, }), ); /* highlight-add-end */ - // .. } ``` @@ -108,6 +113,12 @@ export default async function createPlugin( /* highlight-add-start */ const gitlabProvider = GitlabDiscoveryEntityProvider.fromConfig(env.config, { logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // optional: alternatively, use schedule scheduler: env.scheduler, }); env.eventBroker.subscribe(gitlabProvider); @@ -123,16 +134,18 @@ export default async function createPlugin( To use the discovery provider, you'll need a GitLab integration [set up](locations.md) with a `token`. Then you can add a provider config per group -to the catalog configuration: +to the catalog configuration. -```yaml +> > NOTE: if you are using the New Backend System, the `schedule` has to be setup in the config, as shown below. + +```yaml title="app-config.yaml" catalog: providers: gitlab: yourProviderId: host: gitlab-host # Identifies one of the hosts set up in the integrations branch: main # Optional. Used to discover on a specific branch - fallbackBranch: main # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default + fallbackBranch: master # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default skipForkedRepos: false # Optional. If the project is a fork, skip repository group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index c748fba1ba..206153d53c 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -68,20 +68,27 @@ Add the plugin to the plugin catalog `packages/backend/src/plugins/catalog.ts`: /* highlight-add-next-line */ import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; -const builder = await CatalogBuilder.create(env); -/** ... other processors and/or providers ... */ -/* highlight-add-start */ -builder.addEntityProvider( - ...GitlabOrgDiscoveryEntityProvider.fromConfig(env.config, { - logger: env.logger, - // optional: alternatively, use scheduler with schedule defined in app-config.yaml - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 30 }, - timeout: { minutes: 3 }, +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + /** ... other processors and/or providers ... */ + /* highlight-add-start */ + builder.addEntityProvider( + ...GitlabOrgDiscoveryEntityProvider.fromConfig(env.config, { + logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // optional: alternatively, use schedule + scheduler: env.scheduler, }), - }), -); -/* highlight-add-end */ + ); + /* highlight-add-end */ + // .. +} ``` #### Installation with Events Support @@ -116,6 +123,12 @@ export default async function createPlugin( env.config, { logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // optional: alternatively, use schedule scheduler: env.scheduler, }, ); @@ -145,6 +158,8 @@ amount of data, this can take significant time and resources. The token used must have the `read_api` scope, and the Users and Groups fetched will be those visible to the account which provisioned the token. +> > NOTE: if you are using the New Backend System, the `schedule` has to be setup in the config, as shown below. + ```yaml catalog: providers: @@ -153,7 +168,13 @@ catalog: host: gitlab.com orgEnabled: true group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) + allowInherited: true # Allow groups to be ingested even if there are no direct members. groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } ``` ### Groups From 0f394e7fb3a663e37b9f02c3ffd412af8959d5e6 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Mon, 15 Apr 2024 19:55:21 +0200 Subject: [PATCH 12/15] fix: add role to package.json Signed-off-by: ElaineDeMattosSilvaB --- .../package.json | 96 ++++++++++--------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 5490ac80bc..cbe0fe161f 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -1,4 +1,53 @@ { + "name": "@backstage/plugin-catalog-backend-module-gitlab", + "version": "0.3.15-next.1", + "description": "A Backstage catalog backend module that helps integrate towards GitLab", + "backstage": { + "role": "backend-plugin-module" + }, + "publishConfig": { + "access": "public" + }, + "keywords": [ + "backstage" + ], + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-gitlab" + }, + "license": "Apache-2.0", + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "main": "src/index.ts", + "types": "src/index.ts", + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } + }, + "files": [ + "config.d.ts", + "dist" + ], + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, "dependencies": { "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", @@ -11,7 +60,6 @@ "node-fetch": "^2.6.7", "uuid": "^9.0.0" }, - "description": "A Backstage catalog backend module that helps integrate towards GitLab", "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", @@ -20,49 +68,5 @@ "luxon": "^3.0.0", "msw": "^1.0.0" }, - "exports": { - ".": "./src/index.ts", - "./alpha": "./src/alpha.ts", - "./package.json": "./package.json" - }, - "files": [ - "config.d.ts", - "dist" - ], - "homepage": "https://backstage.io", - "keywords": [ - "backstage" - ], - "license": "Apache-2.0", - "main": "src/index.ts", - "name": "@backstage/plugin-catalog-backend-module-gitlab", - "publishConfig": { - "access": "public" - }, - "repository": { - "directory": "plugins/catalog-backend-module-gitlab", - "type": "git", - "url": "https://github.com/backstage/backstage" - }, - "scripts": { - "build": "backstage-cli package build", - "clean": "backstage-cli package clean", - "lint": "backstage-cli package lint", - "postpack": "backstage-cli package postpack", - "prepack": "backstage-cli package prepack", - "start": "backstage-cli package start", - "test": "backstage-cli package test" - }, - "types": "src/index.ts", - "typesVersions": { - "*": { - "alpha": [ - "src/alpha.ts" - ], - "package.json": [ - "package.json" - ] - } - }, - "version": "0.3.14" + "configSchema": "config.d.ts" } From 72e2339a0bd960a27045c7ed85c58a7c37cb7ba3 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Mon, 15 Apr 2024 20:05:25 +0200 Subject: [PATCH 13/15] fix: add missing dependencies Signed-off-by: ElaineDeMattosSilvaB --- .../package.json | 100 +++++++++--------- yarn.lock | 37 +++++++ 2 files changed, 89 insertions(+), 48 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index cbe0fe161f..99fdde8346 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -1,29 +1,66 @@ { - "name": "@backstage/plugin-catalog-backend-module-gitlab", - "version": "0.3.15-next.1", - "description": "A Backstage catalog backend module that helps integrate towards GitLab", "backstage": { "role": "backend-plugin-module" }, - "publishConfig": { - "access": "public" + "configSchema": "config.d.ts", + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/integration": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", + "@gitbeaker/rest": "^40.0.3", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7", + "uuid": "^9.0.0" }, - "keywords": [ - "backstage" - ], - "homepage": "https://backstage.io", - "repository": { - "type": "git", - "url": "https://github.com/backstage/backstage", - "directory": "plugins/catalog-backend-module-gitlab" + "description": "A Backstage catalog backend module that helps integrate towards GitLab", + "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@backstage/plugin-events-backend-test-utils": "workspace:^", + "@types/lodash": "^4.14.151", + "@types/uuid": "^9.0.0", + "luxon": "^3.0.0", + "msw": "^1.0.0" }, - "license": "Apache-2.0", "exports": { ".": "./src/index.ts", "./alpha": "./src/alpha.ts", "./package.json": "./package.json" }, + "files": [ + "config.d.ts", + "dist" + ], + "homepage": "https://backstage.io", + "keywords": [ + "backstage" + ], + "license": "Apache-2.0", "main": "src/index.ts", + "name": "@backstage/plugin-catalog-backend-module-gitlab", + "publishConfig": { + "access": "public" + }, + "repository": { + "directory": "plugins/catalog-backend-module-gitlab", + "type": "git", + "url": "https://github.com/backstage/backstage" + }, + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "postpack": "backstage-cli package postpack", + "prepack": "backstage-cli package prepack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, "types": "src/index.ts", "typesVersions": { "*": { @@ -35,38 +72,5 @@ ] } }, - "files": [ - "config.d.ts", - "dist" - ], - "scripts": { - "build": "backstage-cli package build", - "clean": "backstage-cli package clean", - "lint": "backstage-cli package lint", - "prepack": "backstage-cli package prepack", - "postpack": "backstage-cli package postpack", - "start": "backstage-cli package start", - "test": "backstage-cli package test" - }, - "dependencies": { - "@backstage/backend-common": "workspace:^", - "@backstage/backend-plugin-api": "workspace:^", - "@backstage/backend-tasks": "workspace:^", - "@backstage/catalog-model": "workspace:^", - "@backstage/config": "workspace:^", - "@backstage/integration": "workspace:^", - "@backstage/plugin-catalog-node": "workspace:^", - "lodash": "^4.17.21", - "node-fetch": "^2.6.7", - "uuid": "^9.0.0" - }, - "devDependencies": { - "@backstage/backend-test-utils": "workspace:^", - "@backstage/cli": "workspace:^", - "@types/lodash": "^4.14.151", - "@types/uuid": "^9.0.0", - "luxon": "^3.0.0", - "msw": "^1.0.0" - }, - "configSchema": "config.d.ts" + "version": "0.3.15-next.1" } diff --git a/yarn.lock b/yarn.lock index 78162415e2..8da6d790d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5598,7 +5598,11 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend-test-utils": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + "@gitbeaker/rest": ^40.0.3 "@types/lodash": ^4.14.151 "@types/uuid": ^9.0.0 lodash: ^4.17.21 @@ -11312,6 +11316,17 @@ __metadata: languageName: node linkType: hard +"@gitbeaker/core@npm:^40.0.3": + version: 40.0.3 + resolution: "@gitbeaker/core@npm:40.0.3" + dependencies: + "@gitbeaker/requester-utils": ^40.0.3 + qs: ^6.11.2 + xcase: ^2.0.1 + checksum: 999cea3f1523eabfae473e8a21849fdd4f6b4d30f0de681e735b238713b198ff9381c35d2ba5949d27ddc7ed5a953b14fc5a6ed38a5d98b05b6a5670a0c69162 + languageName: node + linkType: hard + "@gitbeaker/node@npm:^35.1.0, @gitbeaker/node@npm:^35.8.0": version: 35.8.1 resolution: "@gitbeaker/node@npm:35.8.1" @@ -11348,6 +11363,18 @@ __metadata: languageName: node linkType: hard +"@gitbeaker/requester-utils@npm:^40.0.3": + version: 40.0.3 + resolution: "@gitbeaker/requester-utils@npm:40.0.3" + dependencies: + picomatch-browser: ^2.2.6 + qs: ^6.11.2 + rate-limiter-flexible: ^4.0.0 + xcase: ^2.0.1 + checksum: 1a46dc9c730c29e2229b04ed9bbe6cd87b7e5f16c1d0a5e78106f0287ab3d981ce2b099eb41cdb0f9180bb6b11d25f5f1fa92a91b69682bcd4ac986e58b55d31 + languageName: node + linkType: hard + "@gitbeaker/rest@npm:^39.25.0": version: 39.34.3 resolution: "@gitbeaker/rest@npm:39.34.3" @@ -11358,6 +11385,16 @@ __metadata: languageName: node linkType: hard +"@gitbeaker/rest@npm:^40.0.3": + version: 40.0.3 + resolution: "@gitbeaker/rest@npm:40.0.3" + dependencies: + "@gitbeaker/core": ^40.0.3 + "@gitbeaker/requester-utils": ^40.0.3 + checksum: 6a6427b82ee3357fbe07e1235de7a3c98d50155898180fbe4dd641c8575faa93853f1a39185e18876f84469e1a385ccd9beb26fd95b61c260b81eb0ed342355b + languageName: node + linkType: hard + "@google-cloud/container@npm:^5.0.0": version: 5.12.0 resolution: "@google-cloud/container@npm:5.12.0" From d35b6ba9c21320c3f73360310f1e8e4ae75875a4 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Mon, 15 Apr 2024 21:20:06 +0200 Subject: [PATCH 14/15] chore: add api-report Signed-off-by: ElaineDeMattosSilvaB --- plugins/catalog-backend-module-gitlab/api-report.md | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index 80a9d0b8b7..d7fcd1b94d 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -34,7 +34,6 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { ): GitlabDiscoveryEntityProvider[]; // (undocumented) getProviderName(): string; - // (undocumented) refresh(logger: LoggerService): Promise; } From 45945311c9f5cac34c43acb1f1febcc50d3b0516 Mon Sep 17 00:00:00 2001 From: ElaineDeMattosSilvaB Date: Thu, 18 Apr 2024 09:16:09 +0200 Subject: [PATCH 15/15] feat: improve documentation Signed-off-by: ElaineDeMattosSilvaB --- docs/integrations/gitlab/discovery.md | 2 +- docs/integrations/gitlab/org.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 9a588b3796..91efe05fee 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -150,7 +150,7 @@ catalog: group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything - schedule: # optional; same options as in TaskScheduleDefinition + schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System # supports cron, ISO duration, "human duration" as used in code frequency: { minutes: 30 } # supports ISO duration, "human duration" as used in code diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 206153d53c..14587461e5 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -170,7 +170,7 @@ catalog: group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) allowInherited: true # Allow groups to be ingested even if there are no direct members. groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything - schedule: # optional; same options as in TaskScheduleDefinition + schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System. # supports cron, ISO duration, "human duration" as used in code frequency: { minutes: 30 } # supports ISO duration, "human duration" as used in code