From 5ee1f50a526684fdb76141ece55d745db70a60a4 Mon Sep 17 00:00:00 2001 From: Lokesh Kaki Date: Sun, 15 Mar 2026 19:50:24 -0500 Subject: [PATCH] feat(catalog-backend-module-gitlab): add GitLab SCM event translation and bridge wiring Signed-off-by: Lokesh Kaki --- .../package.json | 1 + .../src/events/GitLabScmEventsBridge.ts | 152 +++++ .../src/events/analyzeGitLabWebhookEvent.ts | 535 ++++++++++++++++++ .../src/index.ts | 1 + ...alogModuleGitlabDiscoveryEntityProvider.ts | 27 +- 5 files changed, 715 insertions(+), 1 deletion(-) create mode 100644 plugins/catalog-backend-module-gitlab/src/events/GitLabScmEventsBridge.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/events/analyzeGitLabWebhookEvent.ts diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index e7b562c4e5..099ffd8832 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -55,6 +55,7 @@ "@backstage/backend-plugin-api": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", diff --git a/plugins/catalog-backend-module-gitlab/src/events/GitLabScmEventsBridge.ts b/plugins/catalog-backend-module-gitlab/src/events/GitLabScmEventsBridge.ts new file mode 100644 index 0000000000..34b9b5c9c2 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/events/GitLabScmEventsBridge.ts @@ -0,0 +1,152 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LoggerService } from '@backstage/backend-plugin-api'; +import { CatalogScmEventsService } from '@backstage/plugin-catalog-node/alpha'; +import { EventParams, EventsService } from '@backstage/plugin-events-node'; +import { analyzeGitLabWebhookEvent } from './analyzeGitLabWebhookEvent'; + +function determineEventType(params: EventParams): string | undefined { + const payload = params.eventPayload; + + if ( + payload && + typeof payload === 'object' && + !Array.isArray(payload) && + typeof (payload as { object_kind?: unknown }).object_kind === 'string' + ) { + return (payload as { object_kind: string }).object_kind; + } + + const eventName = + payload && + typeof payload === 'object' && + !Array.isArray(payload) && + typeof (payload as { event_name?: unknown }).event_name === 'string' + ? (payload as { event_name: string }).event_name + : undefined; + if (eventName) { + return eventName; + } + + const metadataType = params.metadata?.['x-gitlab-event']; + if (typeof metadataType === 'string' && metadataType.trim()) { + return metadataType + .trim() + .toLowerCase() + .replace(/\s+hook$/, '') + .replace(/\s+/g, '_'); + } + + if (params.topic.startsWith('gitlab.')) { + return params.topic.slice('gitlab.'.length); + } + + return undefined; +} + +/** + * Takes GitLab webhook events, analyzes them, and publishes them as catalog + * SCM events that entity providers and others can subscribe to. + */ +export class GitLabScmEventsBridge { + readonly #logger: LoggerService; + readonly #events: EventsService; + readonly #catalogScmEvents: CatalogScmEventsService; + #shuttingDown: boolean; + #pendingPublish: Promise | undefined; + + constructor(options: { + logger: LoggerService; + events: EventsService; + catalogScmEvents: CatalogScmEventsService; + }) { + this.#logger = options.logger; + this.#events = options.events; + this.#catalogScmEvents = options.catalogScmEvents; + this.#shuttingDown = false; + } + + async start() { + await this.#events.subscribe({ + id: 'catalog-gitlab-scm-events-bridge', + topics: ['gitlab'], + onEvent: this.#onEvent.bind(this), + }); + } + + async stop() { + this.#shuttingDown = true; + await this.#pendingPublish; + } + + async #onEvent(params: EventParams): Promise { + const eventType = determineEventType(params); + if (!eventType || !params.eventPayload) { + return; + } + + while (this.#pendingPublish) { + await this.#pendingPublish; + } + + if (this.#shuttingDown) { + this.#logger.warn( + `Skipping GitLab webhook event of type "${eventType}" on topic "${params.topic}" because the bridge is shutting down`, + ); + return; + } + + this.#pendingPublish = Promise.resolve().then(async () => { + try { + const output = await analyzeGitLabWebhookEvent( + eventType, + params.eventPayload, + { + logger: this.#logger, + isRelevantPath: path => + path.endsWith('.yaml') || path.endsWith('.yml'), + }, + ); + + if (output.result === 'ok') { + await this.#catalogScmEvents.publish(output.events); + } else if (output.result === 'ignored') { + this.#logger.debug( + `Skipping GitLab webhook event of type "${eventType}" on topic "${params.topic}" because it is ignored: ${output.reason}`, + ); + } else if (output.result === 'aborted') { + this.#logger.warn( + `Skipping GitLab webhook event of type "${eventType}" on topic "${params.topic}" because it is aborted: ${output.reason}`, + ); + } else if (output.result === 'unsupported-event') { + this.#logger.debug( + `Skipping GitLab webhook event of type "${eventType}" on topic "${params.topic}" because it is unsupported: ${output.event}`, + ); + } + } catch (error) { + this.#logger.warn( + `Failed to handle GitLab webhook event of type "${eventType}"`, + error, + ); + } finally { + this.#pendingPublish = undefined; + } + }); + + await this.#pendingPublish; + } +} \ No newline at end of file diff --git a/plugins/catalog-backend-module-gitlab/src/events/analyzeGitLabWebhookEvent.ts b/plugins/catalog-backend-module-gitlab/src/events/analyzeGitLabWebhookEvent.ts new file mode 100644 index 0000000000..4bea0230f0 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/events/analyzeGitLabWebhookEvent.ts @@ -0,0 +1,535 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LoggerService } from '@backstage/backend-plugin-api'; +import { InputError } from '@backstage/errors'; +import { CatalogScmEvent } from '@backstage/plugin-catalog-node/alpha'; +import { WebhookPushEventSchema } from '@gitbeaker/rest'; + +type StringRecord = Record; + +export interface AnalyzeWebhookEventOptions { + logger: LoggerService; + isRelevantPath: (path: string) => boolean; +} + +export type AnalyzeWebhookEventResult = + | { + result: 'unsupported-event'; + event: string; + } + | { + result: 'ignored'; + reason: string; + } + | { + result: 'aborted'; + reason: string; + } + | { + result: 'ok'; + events: CatalogScmEvent[]; + }; + +type PathState = + | { + type: 'added'; + commitUrl?: string; + } + | { + type: 'removed'; + commitUrl?: string; + } + | { + type: 'modified'; + commitUrl?: string; + } + | { + type: 'renamed'; + fromPath: string; + commitUrl?: string; + } + | { + type: 'changed'; + commitUrl?: string; + }; + +type GitLabPushCommit = { + id?: string; + url?: string; + added?: string[]; + removed?: string[]; + modified?: string[]; +}; + +type ChangeDescriptor = { + from?: unknown; + to?: unknown; + old?: unknown; + new?: unknown; + previous?: unknown; + current?: unknown; + before?: unknown; + after?: unknown; +}; + +type GitLabRepositoryUpdateEvent = { + object_kind?: string; + event_name?: string; + action?: string; + deleted_at?: string | null; + path_with_namespace?: string; + old_path_with_namespace?: string; + project?: { + web_url?: string; + path_with_namespace?: string; + deleted_at?: string | null; + }; + changes?: { + web_url?: ChangeDescriptor; + path_with_namespace?: ChangeDescriptor; + old_path_with_namespace?: ChangeDescriptor; + deleted_at?: ChangeDescriptor; + }; +}; + +function isObject(value: unknown): value is StringRecord { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function asString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +function getFromChange(change?: ChangeDescriptor): string | undefined { + return ( + asString(change?.from) ?? + asString(change?.old) ?? + asString(change?.previous) ?? + asString(change?.before) + ); +} + +function getToChange(change?: ChangeDescriptor): string | undefined { + return ( + asString(change?.to) ?? + asString(change?.new) ?? + asString(change?.current) ?? + asString(change?.after) + ); +} + +function extractBranchName(ref?: string): string | undefined { + if (!ref || !ref.startsWith('refs/heads/')) { + return undefined; + } + return ref.slice('refs/heads/'.length); +} + +function getCommitUrl(commit: GitLabPushCommit, repositoryUrl?: string): + | string + | undefined { + if (commit.url) { + return commit.url; + } + if (commit.id && repositoryUrl) { + return `${repositoryUrl}/-/commit/${commit.id}`; + } + return undefined; +} + +function pathStateToCatalogScmEvent( + path: string, + event: PathState, + repositoryUrl: string, + branch: string, +): CatalogScmEvent { + const toBlobUrl = (p: string) => `${repositoryUrl}/-/blob/${branch}/${p}`; + const context = event.commitUrl ? { commitUrl: event.commitUrl } : undefined; + + switch (event.type) { + case 'added': + return { + type: 'location.created', + url: toBlobUrl(path), + context, + }; + case 'removed': + return { + type: 'location.deleted', + url: toBlobUrl(path), + context, + }; + case 'modified': + return { + type: 'location.updated', + url: toBlobUrl(path), + context, + }; + case 'renamed': + return { + type: 'location.moved', + fromUrl: toBlobUrl(event.fromPath), + toUrl: toBlobUrl(path), + context, + }; + case 'changed': + return { + type: 'location.updated', + url: toBlobUrl(path), + context, + }; + default: + // @ts-expect-error Intentionally expected, to check for exhaustive checking of the types + throw new Error(`Unknown file event type: ${event.type}`); + } +} + +function applyAddedPath( + pathState: Map, + path: string, + commitUrl: string | undefined, +) { + const previous = pathState.get(path); + if (!previous) { + pathState.set(path, { type: 'added', commitUrl }); + return; + } + if (previous.type === 'removed') { + pathState.set(path, { type: 'changed', commitUrl }); + return; + } + pathState.set(path, previous); +} + +function applyRemovedPath( + pathState: Map, + path: string, + commitUrl: string | undefined, +) { + const previous = pathState.get(path); + if (!previous) { + pathState.set(path, { type: 'removed', commitUrl }); + return; + } + if (previous.type === 'added') { + pathState.delete(path); + return; + } + if (previous.type === 'changed') { + pathState.set(path, { type: 'removed', commitUrl }); + return; + } + if (previous.type === 'renamed') { + if (!pathState.has(previous.fromPath)) { + pathState.set(previous.fromPath, { type: 'removed', commitUrl }); + } + pathState.delete(path); + return; + } + pathState.set(path, previous); +} + +function applyModifiedPath( + pathState: Map, + path: string, + commitUrl: string | undefined, +) { + const previous = pathState.get(path); + if (!previous) { + pathState.set(path, { type: 'changed', commitUrl }); + return; + } + if (previous.type === 'removed') { + pathState.set(path, previous); + return; + } + pathState.set(path, previous); +} + +function applyRenamedPath( + pathState: Map, + fromPath: string, + toPath: string, + commitUrl: string | undefined, +) { + const previous = pathState.get(fromPath); + pathState.delete(fromPath); + + if (!previous) { + pathState.set(toPath, { type: 'renamed', fromPath, commitUrl }); + return; + } + if (previous.type === 'added') { + pathState.set(toPath, { type: 'added', commitUrl }); + return; + } + if (previous.type === 'renamed') { + pathState.set(toPath, { + type: 'renamed', + fromPath: previous.fromPath, + commitUrl, + }); + return; + } + pathState.set(toPath, { type: 'renamed', fromPath, commitUrl }); +} + +async function onPushEvent( + event: WebhookPushEventSchema, + options: AnalyzeWebhookEventOptions, +): Promise { + const project = isObject(event.project) ? event.project : undefined; + const repositoryUrl = asString(project?.web_url); + const contextUrl = repositoryUrl ?? ''; + const defaultBranch = asString(project?.default_branch); + + if (defaultBranch) { + const expectedRef = `refs/heads/${defaultBranch}`; + if (event.ref !== expectedRef) { + return { + result: 'ignored', + reason: `GitLab push event did not target the default branch, found "${event.ref}" but expected "${expectedRef}": ${contextUrl}`, + }; + } + } + + const commits = (Array.isArray(event.commits) + ? event.commits + : []) as GitLabPushCommit[]; + + if (!commits.length) { + return { + result: 'ignored', + reason: `GitLab push event did not contain any commits: ${contextUrl}`, + }; + } + + const pathState = new Map(); + let hasRelevantPaths = false; + + for (const commit of commits) { + const commitUrl = getCommitUrl(commit, repositoryUrl); + const added = (commit.added ?? []).filter(options.isRelevantPath); + const modified = (commit.modified ?? []).filter(options.isRelevantPath); + const removed = (commit.removed ?? []).filter(options.isRelevantPath); + + if (added.length || modified.length || removed.length) { + hasRelevantPaths = true; + } + + for (const path of modified) { + applyModifiedPath(pathState, path, commitUrl); + } + + const renamePairs = Math.min(added.length, removed.length); + for (let i = 0; i < renamePairs; i++) { + applyRenamedPath(pathState, removed[i], added[i], commitUrl); + } + + for (const path of added.slice(renamePairs)) { + applyAddedPath(pathState, path, commitUrl); + } + + for (const path of removed.slice(renamePairs)) { + applyRemovedPath(pathState, path, commitUrl); + } + } + + if (!hasRelevantPaths) { + return { + result: 'ignored', + reason: `GitLab push event did not affect any relevant paths: ${contextUrl}`, + }; + } + + if (!repositoryUrl) { + return { + result: 'aborted', + reason: 'GitLab push event did not include project.web_url', + }; + } + + const branch = defaultBranch ?? extractBranchName(event.ref) ?? 'main'; + return { + result: 'ok', + events: Array.from(pathState.entries()).map(([path, e]) => + pathStateToCatalogScmEvent(path, e, repositoryUrl, branch), + ), + }; +} + +function getOrigin(url: string): string | undefined { + try { + return new URL(url).origin; + } catch { + return undefined; + } +} + +function toRepositoryUrl(baseUrl: string, pathWithNamespace: string): string { + return `${baseUrl}/${pathWithNamespace}`; +} + +function getCurrentRepositoryUrl( + event: GitLabRepositoryUpdateEvent, +): string | undefined { + const projectUrl = asString(event.project?.web_url); + if (projectUrl) { + return projectUrl; + } + + return getToChange(event.changes?.web_url); +} + +function getPreviousRepositoryUrl( + event: GitLabRepositoryUpdateEvent, + currentRepositoryUrl?: string, +): string | undefined { + const changedUrl = getFromChange(event.changes?.web_url); + if (changedUrl) { + return changedUrl; + } + + const oldPathWithNamespace = + asString(event.old_path_with_namespace) ?? + getFromChange(event.changes?.path_with_namespace) ?? + getFromChange(event.changes?.old_path_with_namespace); + if (!oldPathWithNamespace) { + return undefined; + } + + const projectPathWithNamespace = asString(event.project?.path_with_namespace); + const projectUrl = asString(event.project?.web_url); + + if ( + currentRepositoryUrl && + projectPathWithNamespace && + currentRepositoryUrl.endsWith(`/${projectPathWithNamespace}`) + ) { + const prefix = currentRepositoryUrl.slice( + 0, + -projectPathWithNamespace.length - 1, + ); + return toRepositoryUrl(prefix, oldPathWithNamespace); + } + + const baseUrl = + (projectUrl && getOrigin(projectUrl)) || + (currentRepositoryUrl && getOrigin(currentRepositoryUrl)); + if (!baseUrl) { + return undefined; + } + + return toRepositoryUrl(baseUrl, oldPathWithNamespace); +} + +function isRepositoryDeletionEvent(event: GitLabRepositoryUpdateEvent): boolean { + const eventName = asString(event.event_name)?.toLowerCase() ?? ''; + const action = asString(event.action)?.toLowerCase() ?? ''; + + if ( + eventName.includes('destroy') || + eventName.includes('delete') || + action.includes('destroy') || + action.includes('delete') || + action.includes('remove') + ) { + return true; + } + + if (event.deleted_at || event.project?.deleted_at) { + return true; + } + + return Boolean(getToChange(event.changes?.deleted_at)); +} + +async function onRepositoryUpdateEvent( + event: GitLabRepositoryUpdateEvent, +): Promise { + const currentRepositoryUrl = getCurrentRepositoryUrl(event); + const previousRepositoryUrl = getPreviousRepositoryUrl( + event, + currentRepositoryUrl, + ); + + if (isRepositoryDeletionEvent(event)) { + const repositoryUrl = currentRepositoryUrl ?? previousRepositoryUrl; + if (!repositoryUrl) { + return { + result: 'ignored', + reason: + 'GitLab repository_update event did not include sufficient data for repository deletion handling', + }; + } + + return { + result: 'ok', + events: [ + { + type: 'repository.deleted', + url: repositoryUrl, + }, + ], + }; + } + + if ( + previousRepositoryUrl && + currentRepositoryUrl && + previousRepositoryUrl !== currentRepositoryUrl + ) { + return { + result: 'ok', + events: [ + { + type: 'repository.moved', + fromUrl: previousRepositoryUrl, + toUrl: currentRepositoryUrl, + }, + ], + }; + } + + return { + result: 'ignored', + reason: 'GitLab repository_update event did not contain supported changes', + }; +} + +export async function analyzeGitLabWebhookEvent( + eventType: string, + eventPayload: unknown, + options: AnalyzeWebhookEventOptions, +): Promise { + if (!isObject(eventPayload)) { + throw new InputError('GitLab webhook event payload is not an object'); + } + + if (eventType === 'push') { + return onPushEvent(eventPayload as WebhookPushEventSchema, options); + } + + if (eventType === 'repository_update') { + return onRepositoryUpdateEvent(eventPayload as GitLabRepositoryUpdateEvent); + } + + return { + result: 'unsupported-event', + event: eventType, + }; +} \ No newline at end of file diff --git a/plugins/catalog-backend-module-gitlab/src/index.ts b/plugins/catalog-backend-module-gitlab/src/index.ts index 4f64e76395..0ad28b8a4b 100644 --- a/plugins/catalog-backend-module-gitlab/src/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/index.ts @@ -26,6 +26,7 @@ export { GitlabDiscoveryEntityProvider, GitlabOrgDiscoveryEntityProvider, } from './providers'; +export { analyzeGitLabWebhookEvent } from './events/analyzeGitLabWebhookEvent'; export type { GitLabUser, GitLabGroup, diff --git a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts index 4f9626a130..d776e6ecd6 100644 --- a/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts @@ -19,7 +19,9 @@ import { createBackendModule, } from '@backstage/backend-plugin-api'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { catalogScmEventsServiceRef } from '@backstage/plugin-catalog-node/alpha'; import { eventsServiceRef } from '@backstage/plugin-events-node'; +import { GitLabScmEventsBridge } from '../events/GitLabScmEventsBridge'; import { GitlabDiscoveryEntityProvider } from '../providers'; /** @@ -36,11 +38,21 @@ export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({ deps: { config: coreServices.rootConfig, catalog: catalogProcessingExtensionPoint, + catalogScmEvents: catalogScmEventsServiceRef, logger: coreServices.logger, scheduler: coreServices.scheduler, events: eventsServiceRef, + lifecycle: coreServices.lifecycle, }, - async init({ config, catalog, logger, scheduler, events }) { + async init({ + config, + catalog, + catalogScmEvents, + logger, + scheduler, + events, + lifecycle, + }) { const gitlabDiscoveryEntityProvider = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, @@ -48,6 +60,19 @@ export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({ scheduler, }); catalog.addEntityProvider(gitlabDiscoveryEntityProvider); + + const bridge = new GitLabScmEventsBridge({ + logger, + events, + catalogScmEvents, + }); + + lifecycle.addStartupHook(async () => { + await bridge.start(); + }); + lifecycle.addShutdownHook(async () => { + await bridge.stop(); + }); }, }); },