diff --git a/.changeset/early-cases-laugh.md b/.changeset/early-cases-laugh.md new file mode 100644 index 0000000000..ba03c156f3 --- /dev/null +++ b/.changeset/early-cases-laugh.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Implemented `catalogScmEventsServiceRef` event handling in the builtin entity providers. diff --git a/.changeset/thirty-dancers-clean.md b/.changeset/thirty-dancers-clean.md new file mode 100644 index 0000000000..87d15a9039 --- /dev/null +++ b/.changeset/thirty-dancers-clean.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Implemented translation of webhook events into `catalogScmEventsServiceRef` events. diff --git a/.changeset/warm-colts-admire.md b/.changeset/warm-colts-admire.md new file mode 100644 index 0000000000..08fe3de363 --- /dev/null +++ b/.changeset/warm-colts-admire.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-node': minor +--- + +Introduced the `catalogScmEventsServiceRef`, along with `CatalogScmEventsService` and associated types. These allow communicating a unified set of events, that parts of the catalog can react to. diff --git a/plugins/catalog-backend-module-github/dev/index.ts b/plugins/catalog-backend-module-github/dev/index.ts new file mode 100644 index 0000000000..c8d2cb1798 --- /dev/null +++ b/plugins/catalog-backend-module-github/dev/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createBackend } from '@backstage/backend-defaults'; + +const backend = createBackend(); +backend.add(import('@backstage/plugin-events-backend')); +backend.add(import('@backstage/plugin-events-backend-module-github')); +backend.add(import('@backstage/plugin-events-backend-module-google-pubsub')); +backend.add(import('@backstage/plugin-catalog-backend')); +backend.add(import('../src')); +backend.start(); diff --git a/plugins/catalog-backend-module-github/knip-report.md b/plugins/catalog-backend-module-github/knip-report.md index c9e8569b52..2661c35327 100644 --- a/plugins/catalog-backend-module-github/knip-report.md +++ b/plugins/catalog-backend-module-github/knip-report.md @@ -1,12 +1,2 @@ # Knip report - -## Unlisted dependencies (4) - -| Name | Location | Severity | -| :---------------------- | :-------------------------------------------- | :------- | -| @octokit/webhooks-types | plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts | error | -| @octokit/webhooks-types | plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts | error | -| @octokit/webhooks-types | plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts | error | -| @octokit/webhooks-types | plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts | error | - diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json index d3a9d20ad6..49ecf76f57 100644 --- a/plugins/catalog-backend-module-github/package.json +++ b/plugins/catalog-backend-module-github/package.json @@ -54,22 +54,32 @@ "@backstage/backend-plugin-api": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-events-node": "workspace:^", + "@backstage/types": "workspace:^", + "@octokit/auth-callback": "^5.0.0", "@octokit/core": "^5.2.0", "@octokit/graphql": "^7.0.2", "@octokit/plugin-throttling": "^8.1.3", "@octokit/rest": "^19.0.3", + "@octokit/webhooks-types": "^7.6.1", "git-url-parse": "^15.0.0", "lodash": "^4.17.21", "minimatch": "^9.0.0", + "octokit": "^3.0.0", "uuid": "^11.0.0" }, "devDependencies": { + "@backstage/backend-defaults": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", + "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-events-backend": "workspace:^", + "@backstage/plugin-events-backend-module-github": "workspace:^", + "@backstage/plugin-events-backend-module-google-pubsub": "workspace:^", "@types/lodash": "^4.14.151", "msw": "^2.0.0", "type-fest": "^4.41.0" diff --git a/plugins/catalog-backend-module-github/src/events/GithubScmEventsBridge.ts b/plugins/catalog-backend-module-github/src/events/GithubScmEventsBridge.ts new file mode 100644 index 0000000000..e483efa267 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/events/GithubScmEventsBridge.ts @@ -0,0 +1,119 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LoggerService } from '@backstage/backend-plugin-api'; +import { CatalogScmEventsService } from '@backstage/plugin-catalog-node/alpha'; +import { EventParams, EventsService } from '@backstage/plugin-events-node'; +import { analyzeGithubWebhookEvent } from './analyzeGithubWebhookEvent'; +import { OctokitProviderService } from '../util/octokitProviderService'; + +/** + * Takes GitHub webhook events, analyzes them, and publishes them as catalog SCM + * events that entity providers and others can subscribe to. + */ +export class GithubScmEventsBridge { + readonly #logger: LoggerService; + readonly #events: EventsService; + readonly #octokitProvider: OctokitProviderService; + readonly #catalogScmEvents: CatalogScmEventsService; + #shuttingDown: boolean; + #pendingPublish: Promise | undefined; + + constructor(options: { + logger: LoggerService; + events: EventsService; + octokitProvider: OctokitProviderService; + catalogScmEvents: CatalogScmEventsService; + }) { + this.#logger = options.logger; + this.#events = options.events; + this.#octokitProvider = options.octokitProvider; + this.#catalogScmEvents = options.catalogScmEvents; + this.#shuttingDown = false; + } + + async start() { + await this.#events.subscribe({ + id: 'catalog-github-scm-events-bridge', + topics: ['github'], + onEvent: this.#onEvent.bind(this), + }); + } + + async stop() { + this.#shuttingDown = true; + await this.#pendingPublish; + } + + async #onEvent(params: EventParams): Promise { + const eventType = params.metadata?.['x-github-event'] as string | undefined; + const eventPayload = params.eventPayload; + if (!eventType || !eventPayload) { + return; + } + + while (this.#pendingPublish) { + await this.#pendingPublish; + } + + if (this.#shuttingDown) { + this.#logger.warn( + `Skipping GitHub webhook event of type "${params.topic}" because the bridge is shutting down`, + ); + return; + } + + this.#pendingPublish = Promise.resolve().then(async () => { + try { + const output = await analyzeGithubWebhookEvent( + eventType, + eventPayload, + { + octokitProvider: this.#octokitProvider, + logger: this.#logger, + isRelevantPath: path => + path.endsWith('.yaml') || path.endsWith('.yml'), + }, + ); + + if (output.result === 'ok') { + await this.#catalogScmEvents.publish(output.events); + } else if (output.result === 'ignored') { + this.#logger.debug( + `Skipping GitHub webhook event of type "${params.topic}" because it is ignored: ${output.reason}`, + ); + } else if (output.result === 'aborted') { + this.#logger.warn( + `Skipping GitHub webhook event of type "${params.topic}" because it is aborted: ${output.reason}`, + ); + } else if (output.result === 'unsupported-event') { + this.#logger.debug( + `Skipping GitHub webhook event of type "${params.topic}" because it is unsupported: ${output.event}`, + ); + } + } catch (error) { + this.#logger.warn( + `Failed to handle GitHub webhook event of type "${eventType}"`, + error, + ); + } finally { + this.#pendingPublish = undefined; + } + }); + + await this.#pendingPublish; + } +} diff --git a/plugins/catalog-backend-module-github/src/events/__fixtures__/push-event-complex.json b/plugins/catalog-backend-module-github/src/events/__fixtures__/push-event-complex.json new file mode 100644 index 0000000000..b3a210706d --- /dev/null +++ b/plugins/catalog-backend-module-github/src/events/__fixtures__/push-event-complex.json @@ -0,0 +1,233 @@ +{ + "ref": "refs/heads/master", + "before": "b8bc5e5c7b2d859cd173f08db3880bd3a8d31f6a", + "after": "677c191848d92449c3d978becc71a59ea3e9777e", + "repository": { + "id": 125428, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjU0Mjg=", + "name": "example-repo", + "full_name": "example-owner/example-repo", + "private": false, + "owner": { + "name": "example-owner", + "email": null, + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/example-owner", + "html_url": "https://ghe.example.net/example-owner", + "followers_url": "https://ghe.example.net/api/v3/users/example-owner/followers", + "following_url": "https://ghe.example.net/api/v3/users/example-owner/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/example-owner/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/example-owner/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/example-owner/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/example-owner/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/users/example-owner/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/example-owner/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://ghe.example.net/example-owner/example-repo", + "description": null, + "fork": false, + "url": "https://ghe.example.net/example-owner/example-repo", + "forks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/forks", + "keys_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/keys{/key_id}", + "collaborators_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/collaborators{/collaborator}", + "teams_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/teams", + "hooks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/hooks", + "issue_events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/events{/number}", + "events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/events", + "assignees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/assignees{/user}", + "branches_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/branches{/branch}", + "tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/tags", + "blobs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/blobs{/sha}", + "git_tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/tags{/sha}", + "git_refs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/refs{/sha}", + "trees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/trees{/sha}", + "statuses_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/statuses/{sha}", + "languages_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/languages", + "stargazers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/stargazers", + "contributors_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contributors", + "subscribers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscribers", + "subscription_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscription", + "commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/commits{/sha}", + "git_commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/commits{/sha}", + "comments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/comments{/number}", + "issue_comment_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/comments{/number}", + "contents_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contents/{+path}", + "compare_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/compare/{base}...{head}", + "merges_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/merges", + "archive_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/{archive_format}{/ref}", + "downloads_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/downloads", + "issues_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues{/number}", + "pulls_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/pulls{/number}", + "milestones_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/milestones{/number}", + "notifications_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/notifications{?since,all,participating}", + "labels_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/labels{/name}", + "releases_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/releases{/id}", + "deployments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/deployments", + "created_at": 1583321868, + "updated_at": "2022-09-08T05:09:21Z", + "pushed_at": 1741769053, + "git_url": "git://ghe.example.net/example-owner/example-repo.git", + "ssh_url": "git@ghe.example.net:example-owner/example-repo.git", + "clone_url": "https://ghe.example.net/example-owner/example-repo.git", + "svn_url": "https://ghe.example.net/example-owner/example-repo", + "homepage": null, + "size": 38734, + "stargazers_count": 0, + "watchers_count": 0, + "language": "TypeScript", + "has_issues": true, + "has_projects": true, + "has_downloads": false, + "has_wiki": false, + "has_pages": false, + "has_discussions": false, + "forks_count": 2, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 8, + "license": null, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 2, + "open_issues": 8, + "watchers": 0, + "default_branch": "master", + "stargazers": 0, + "master_branch": "master", + "organization": "example-owner", + "custom_properties": {} + }, + "pusher": { + "name": "john", + "email": "john@example.com" + }, + "organization": { + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "url": "https://ghe.example.net/api/v3/orgs/example-owner", + "repos_url": "https://ghe.example.net/api/v3/orgs/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/orgs/example-owner/events", + "hooks_url": "https://ghe.example.net/api/v3/orgs/example-owner/hooks", + "issues_url": "https://ghe.example.net/api/v3/orgs/example-owner/issues", + "members_url": "https://ghe.example.net/api/v3/orgs/example-owner/members{/member}", + "public_members_url": "https://ghe.example.net/api/v3/orgs/example-owner/public_members{/member}", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "description": null + }, + "enterprise": { + "id": 1, + "slug": "example", + "name": "Example", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://ghe.example.net/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://ghe.example.net/enterprises/example", + "created_at": "2019-03-30T15:03:21Z", + "updated_at": "2024-12-09T09:09:26Z" + }, + "sender": { + "login": "john", + "id": 11272, + "node_id": "MDQ6VXNlcjExMjcy", + "avatar_url": "https://ghe.example.net/avatars/u/11272?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/john", + "html_url": "https://ghe.example.net/john", + "followers_url": "https://ghe.example.net/api/v3/users/john/followers", + "following_url": "https://ghe.example.net/api/v3/users/john/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/john/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/john/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/john/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/john/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/john/repos", + "events_url": "https://ghe.example.net/api/v3/users/john/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/john/received_events", + "type": "User", + "site_admin": false + }, + "installation": { + "id": 56902, + "node_id": "MDIzOkludGVncmF0aW9uSW5zdGFsbGF0aW9uNTY5MDI=" + }, + "created": false, + "deleted": false, + "forced": false, + "base_ref": null, + "compare": "https://ghe.example.net/example-owner/example-repo/compare/b8bc5e5c7b2d...677c191848d9", + "commits": [ + { + "id": "8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + "tree_id": "5565855e61695f86690f3bdb3d1919f0b784e5eb", + "distinct": false, + "message": "Perform some important action", + "timestamp": "2025-03-12T14:13:10+05:30", + "url": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + "author": { + "name": "Betty Binomial", + "email": "betty@example.com", + "username": "betty" + }, + "committer": { + "name": "Betty Binomial", + "email": "betty@example.com", + "username": "betty" + }, + "added": [], + "removed": ["old/catalog-info.yaml"], + "modified": [] + }, + { + "id": "677c191848d92449c3d978becc71a59ea3e9777e", + "tree_id": "5565855e61695f86690f3bdb3d1919f0b784e5eb", + "distinct": true, + "message": "Merge pull request #1457 from example-owner/betty/patch-kkev\n\nPerform some important action", + "timestamp": "2025-03-12T14:14:13+05:30", + "url": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + "author": { + "name": "John Doe", + "email": "john@example.com", + "username": "john" + }, + "committer": { + "name": "GitHub Enterprise", + "email": "noreply+ghe@example.net" + }, + "added": ["new/catalog-info.yaml"], + "removed": [], + "modified": [] + } + ], + "head_commit": { + "id": "677c191848d92449c3d978becc71a59ea3e9777e", + "tree_id": "5565855e61695f86690f3bdb3d1919f0b784e5eb", + "distinct": true, + "message": "Merge pull request #1457 from example-owner/betty/patch-kkev\n\nPerform some important action", + "timestamp": "2025-03-12T14:14:13+05:30", + "url": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + "author": { + "name": "John Doe", + "email": "john@example.com", + "username": "john" + }, + "committer": { + "name": "GitHub Enterprise", + "email": "noreply+ghe@example.net" + }, + "added": [], + "removed": [], + "modified": ["build-info.yaml"] + } +} diff --git a/plugins/catalog-backend-module-github/src/events/__fixtures__/push-event-simple.json b/plugins/catalog-backend-module-github/src/events/__fixtures__/push-event-simple.json new file mode 100644 index 0000000000..865c381096 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/events/__fixtures__/push-event-simple.json @@ -0,0 +1,233 @@ +{ + "ref": "refs/heads/master", + "before": "b8bc5e5c7b2d859cd173f08db3880bd3a8d31f6a", + "after": "677c191848d92449c3d978becc71a59ea3e9777e", + "repository": { + "id": 125428, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjU0Mjg=", + "name": "example-repo", + "full_name": "example-owner/example-repo", + "private": false, + "owner": { + "name": "example-owner", + "email": null, + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/example-owner", + "html_url": "https://ghe.example.net/example-owner", + "followers_url": "https://ghe.example.net/api/v3/users/example-owner/followers", + "following_url": "https://ghe.example.net/api/v3/users/example-owner/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/example-owner/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/example-owner/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/example-owner/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/example-owner/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/users/example-owner/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/example-owner/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://ghe.example.net/example-owner/example-repo", + "description": null, + "fork": false, + "url": "https://ghe.example.net/example-owner/example-repo", + "forks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/forks", + "keys_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/keys{/key_id}", + "collaborators_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/collaborators{/collaborator}", + "teams_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/teams", + "hooks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/hooks", + "issue_events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/events{/number}", + "events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/events", + "assignees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/assignees{/user}", + "branches_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/branches{/branch}", + "tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/tags", + "blobs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/blobs{/sha}", + "git_tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/tags{/sha}", + "git_refs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/refs{/sha}", + "trees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/trees{/sha}", + "statuses_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/statuses/{sha}", + "languages_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/languages", + "stargazers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/stargazers", + "contributors_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contributors", + "subscribers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscribers", + "subscription_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscription", + "commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/commits{/sha}", + "git_commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/commits{/sha}", + "comments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/comments{/number}", + "issue_comment_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/comments{/number}", + "contents_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contents/{+path}", + "compare_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/compare/{base}...{head}", + "merges_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/merges", + "archive_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/{archive_format}{/ref}", + "downloads_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/downloads", + "issues_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues{/number}", + "pulls_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/pulls{/number}", + "milestones_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/milestones{/number}", + "notifications_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/notifications{?since,all,participating}", + "labels_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/labels{/name}", + "releases_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/releases{/id}", + "deployments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/deployments", + "created_at": 1583321868, + "updated_at": "2022-09-08T05:09:21Z", + "pushed_at": 1741769053, + "git_url": "git://ghe.example.net/example-owner/example-repo.git", + "ssh_url": "git@ghe.example.net:example-owner/example-repo.git", + "clone_url": "https://ghe.example.net/example-owner/example-repo.git", + "svn_url": "https://ghe.example.net/example-owner/example-repo", + "homepage": null, + "size": 38734, + "stargazers_count": 0, + "watchers_count": 0, + "language": "TypeScript", + "has_issues": true, + "has_projects": true, + "has_downloads": false, + "has_wiki": false, + "has_pages": false, + "has_discussions": false, + "forks_count": 2, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 8, + "license": null, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 2, + "open_issues": 8, + "watchers": 0, + "default_branch": "master", + "stargazers": 0, + "master_branch": "master", + "organization": "example-owner", + "custom_properties": {} + }, + "pusher": { + "name": "john", + "email": "john@example.com" + }, + "organization": { + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "url": "https://ghe.example.net/api/v3/orgs/example-owner", + "repos_url": "https://ghe.example.net/api/v3/orgs/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/orgs/example-owner/events", + "hooks_url": "https://ghe.example.net/api/v3/orgs/example-owner/hooks", + "issues_url": "https://ghe.example.net/api/v3/orgs/example-owner/issues", + "members_url": "https://ghe.example.net/api/v3/orgs/example-owner/members{/member}", + "public_members_url": "https://ghe.example.net/api/v3/orgs/example-owner/public_members{/member}", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "description": null + }, + "enterprise": { + "id": 1, + "slug": "example", + "name": "Example", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://ghe.example.net/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://ghe.example.net/enterprises/example", + "created_at": "2019-03-30T15:03:21Z", + "updated_at": "2024-12-09T09:09:26Z" + }, + "sender": { + "login": "john", + "id": 11272, + "node_id": "MDQ6VXNlcjExMjcy", + "avatar_url": "https://ghe.example.net/avatars/u/11272?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/john", + "html_url": "https://ghe.example.net/john", + "followers_url": "https://ghe.example.net/api/v3/users/john/followers", + "following_url": "https://ghe.example.net/api/v3/users/john/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/john/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/john/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/john/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/john/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/john/repos", + "events_url": "https://ghe.example.net/api/v3/users/john/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/john/received_events", + "type": "User", + "site_admin": false + }, + "installation": { + "id": 56902, + "node_id": "MDIzOkludGVncmF0aW9uSW5zdGFsbGF0aW9uNTY5MDI=" + }, + "created": false, + "deleted": false, + "forced": false, + "base_ref": null, + "compare": "https://ghe.example.net/example-owner/example-repo/compare/b8bc5e5c7b2d...677c191848d9", + "commits": [ + { + "id": "8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + "tree_id": "5565855e61695f86690f3bdb3d1919f0b784e5eb", + "distinct": false, + "message": "Perform some important action", + "timestamp": "2025-03-12T14:13:10+05:30", + "url": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + "author": { + "name": "Betty Binomial", + "email": "betty@example.com", + "username": "betty" + }, + "committer": { + "name": "Betty Binomial", + "email": "betty@example.com", + "username": "betty" + }, + "added": [], + "removed": [], + "modified": ["catalog-info.yaml"] + }, + { + "id": "677c191848d92449c3d978becc71a59ea3e9777e", + "tree_id": "5565855e61695f86690f3bdb3d1919f0b784e5eb", + "distinct": true, + "message": "Merge pull request #1457 from example-owner/betty/patch-kkev\n\nPerform some important action", + "timestamp": "2025-03-12T14:14:13+05:30", + "url": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + "author": { + "name": "John Doe", + "email": "john@example.com", + "username": "john" + }, + "committer": { + "name": "GitHub Enterprise", + "email": "noreply+ghe@example.net" + }, + "added": [], + "removed": ["catalog-info-2.yaml", "catalog-info-3.yaml"], + "modified": ["some-other-file.txt"] + } + ], + "head_commit": { + "id": "677c191848d92449c3d978becc71a59ea3e9777e", + "tree_id": "5565855e61695f86690f3bdb3d1919f0b784e5eb", + "distinct": true, + "message": "Merge pull request #1457 from example-owner/betty/patch-kkev\n\nPerform some important action", + "timestamp": "2025-03-12T14:14:13+05:30", + "url": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + "author": { + "name": "John Doe", + "email": "john@example.com", + "username": "john" + }, + "committer": { + "name": "GitHub Enterprise", + "email": "noreply+ghe@example.net" + }, + "added": [], + "removed": [], + "modified": ["build-info.yaml"] + } +} diff --git a/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-archived-event.json b/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-archived-event.json new file mode 100644 index 0000000000..4a063d0812 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-archived-event.json @@ -0,0 +1,159 @@ +{ + "action": "archived", + "repository": { + "id": 125428, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjU0Mjg=", + "name": "example-repo", + "full_name": "example-owner/example-repo", + "private": false, + "owner": { + "name": "example-owner", + "email": null, + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/example-owner", + "html_url": "https://ghe.example.net/example-owner", + "followers_url": "https://ghe.example.net/api/v3/users/example-owner/followers", + "following_url": "https://ghe.example.net/api/v3/users/example-owner/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/example-owner/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/example-owner/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/example-owner/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/example-owner/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/users/example-owner/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/example-owner/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://ghe.example.net/example-owner/example-repo", + "description": null, + "fork": false, + "url": "https://ghe.example.net/example-owner/example-repo", + "forks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/forks", + "keys_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/keys{/key_id}", + "collaborators_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/collaborators{/collaborator}", + "teams_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/teams", + "hooks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/hooks", + "issue_events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/events{/number}", + "events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/events", + "assignees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/assignees{/user}", + "branches_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/branches{/branch}", + "tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/tags", + "blobs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/blobs{/sha}", + "git_tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/tags{/sha}", + "git_refs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/refs{/sha}", + "trees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/trees{/sha}", + "statuses_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/statuses/{sha}", + "languages_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/languages", + "stargazers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/stargazers", + "contributors_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contributors", + "subscribers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscribers", + "subscription_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscription", + "commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/commits{/sha}", + "git_commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/commits{/sha}", + "comments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/comments{/number}", + "issue_comment_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/comments{/number}", + "contents_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contents/{+path}", + "compare_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/compare/{base}...{head}", + "merges_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/merges", + "archive_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/{archive_format}{/ref}", + "downloads_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/downloads", + "issues_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues{/number}", + "pulls_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/pulls{/number}", + "milestones_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/milestones{/number}", + "notifications_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/notifications{?since,all,participating}", + "labels_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/labels{/name}", + "releases_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/releases{/id}", + "deployments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/deployments", + "created_at": 1583321868, + "updated_at": "2022-09-08T05:09:21Z", + "pushed_at": 1741769053, + "git_url": "git://ghe.example.net/example-owner/example-repo.git", + "ssh_url": "git@ghe.example.net:example-owner/example-repo.git", + "clone_url": "https://ghe.example.net/example-owner/example-repo.git", + "svn_url": "https://ghe.example.net/example-owner/example-repo", + "homepage": null, + "size": 38734, + "stargazers_count": 0, + "watchers_count": 0, + "language": "TypeScript", + "has_issues": true, + "has_projects": true, + "has_downloads": false, + "has_wiki": false, + "has_pages": false, + "has_discussions": false, + "forks_count": 2, + "mirror_url": null, + "archived": true, + "disabled": false, + "open_issues_count": 8, + "license": null, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 2, + "open_issues": 8, + "watchers": 0, + "default_branch": "master", + "stargazers": 0, + "master_branch": "master", + "organization": "example-owner", + "custom_properties": {} + }, + "organization": { + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "url": "https://ghe.example.net/api/v3/orgs/example-owner", + "repos_url": "https://ghe.example.net/api/v3/orgs/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/orgs/example-owner/events", + "hooks_url": "https://ghe.example.net/api/v3/orgs/example-owner/hooks", + "issues_url": "https://ghe.example.net/api/v3/orgs/example-owner/issues", + "members_url": "https://ghe.example.net/api/v3/orgs/example-owner/members{/member}", + "public_members_url": "https://ghe.example.net/api/v3/orgs/example-owner/public_members{/member}", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "description": null + }, + "enterprise": { + "id": 1, + "slug": "example", + "name": "Example", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://ghe.example.net/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://ghe.example.net/enterprises/example", + "created_at": "2019-03-30T15:03:21Z", + "updated_at": "2024-12-09T09:09:26Z" + }, + "sender": { + "login": "john", + "id": 11272, + "node_id": "MDQ6VXNlcjExMjcy", + "avatar_url": "https://ghe.example.net/avatars/u/11272?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/john", + "html_url": "https://ghe.example.net/john", + "followers_url": "https://ghe.example.net/api/v3/users/john/followers", + "following_url": "https://ghe.example.net/api/v3/users/john/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/john/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/john/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/john/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/john/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/john/repos", + "events_url": "https://ghe.example.net/api/v3/users/john/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/john/received_events", + "type": "User", + "site_admin": false + }, + "installation": { + "id": 56902, + "node_id": "MDIzOkludGVncmF0aW9uSW5zdGFsbGF0aW9uNTY5MDI=" + } +} diff --git a/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-deleted-event.json b/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-deleted-event.json new file mode 100644 index 0000000000..b43e0d8020 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-deleted-event.json @@ -0,0 +1,159 @@ +{ + "action": "deleted", + "repository": { + "id": 125428, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjU0Mjg=", + "name": "example-repo", + "full_name": "example-owner/example-repo", + "private": false, + "owner": { + "name": "example-owner", + "email": null, + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/example-owner", + "html_url": "https://ghe.example.net/example-owner", + "followers_url": "https://ghe.example.net/api/v3/users/example-owner/followers", + "following_url": "https://ghe.example.net/api/v3/users/example-owner/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/example-owner/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/example-owner/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/example-owner/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/example-owner/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/users/example-owner/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/example-owner/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://ghe.example.net/example-owner/example-repo", + "description": null, + "fork": false, + "url": "https://ghe.example.net/example-owner/example-repo", + "forks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/forks", + "keys_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/keys{/key_id}", + "collaborators_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/collaborators{/collaborator}", + "teams_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/teams", + "hooks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/hooks", + "issue_events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/events{/number}", + "events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/events", + "assignees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/assignees{/user}", + "branches_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/branches{/branch}", + "tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/tags", + "blobs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/blobs{/sha}", + "git_tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/tags{/sha}", + "git_refs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/refs{/sha}", + "trees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/trees{/sha}", + "statuses_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/statuses/{sha}", + "languages_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/languages", + "stargazers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/stargazers", + "contributors_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contributors", + "subscribers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscribers", + "subscription_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscription", + "commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/commits{/sha}", + "git_commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/commits{/sha}", + "comments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/comments{/number}", + "issue_comment_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/comments{/number}", + "contents_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contents/{+path}", + "compare_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/compare/{base}...{head}", + "merges_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/merges", + "archive_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/{archive_format}{/ref}", + "downloads_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/downloads", + "issues_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues{/number}", + "pulls_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/pulls{/number}", + "milestones_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/milestones{/number}", + "notifications_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/notifications{?since,all,participating}", + "labels_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/labels{/name}", + "releases_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/releases{/id}", + "deployments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/deployments", + "created_at": 1583321868, + "updated_at": "2022-09-08T05:09:21Z", + "pushed_at": 1741769053, + "git_url": "git://ghe.example.net/example-owner/example-repo.git", + "ssh_url": "git@ghe.example.net:example-owner/example-repo.git", + "clone_url": "https://ghe.example.net/example-owner/example-repo.git", + "svn_url": "https://ghe.example.net/example-owner/example-repo", + "homepage": null, + "size": 38734, + "stargazers_count": 0, + "watchers_count": 0, + "language": "TypeScript", + "has_issues": true, + "has_projects": true, + "has_downloads": false, + "has_wiki": false, + "has_pages": false, + "has_discussions": false, + "forks_count": 2, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 8, + "license": null, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 2, + "open_issues": 8, + "watchers": 0, + "default_branch": "master", + "stargazers": 0, + "master_branch": "master", + "organization": "example-owner", + "custom_properties": {} + }, + "organization": { + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "url": "https://ghe.example.net/api/v3/orgs/example-owner", + "repos_url": "https://ghe.example.net/api/v3/orgs/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/orgs/example-owner/events", + "hooks_url": "https://ghe.example.net/api/v3/orgs/example-owner/hooks", + "issues_url": "https://ghe.example.net/api/v3/orgs/example-owner/issues", + "members_url": "https://ghe.example.net/api/v3/orgs/example-owner/members{/member}", + "public_members_url": "https://ghe.example.net/api/v3/orgs/example-owner/public_members{/member}", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "description": null + }, + "enterprise": { + "id": 1, + "slug": "example", + "name": "Example", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://ghe.example.net/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://ghe.example.net/enterprises/example", + "created_at": "2019-03-30T15:03:21Z", + "updated_at": "2024-12-09T09:09:26Z" + }, + "sender": { + "login": "john", + "id": 11272, + "node_id": "MDQ6VXNlcjExMjcy", + "avatar_url": "https://ghe.example.net/avatars/u/11272?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/john", + "html_url": "https://ghe.example.net/john", + "followers_url": "https://ghe.example.net/api/v3/users/john/followers", + "following_url": "https://ghe.example.net/api/v3/users/john/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/john/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/john/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/john/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/john/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/john/repos", + "events_url": "https://ghe.example.net/api/v3/users/john/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/john/received_events", + "type": "User", + "site_admin": false + }, + "installation": { + "id": 56902, + "node_id": "MDIzOkludGVncmF0aW9uSW5zdGFsbGF0aW9uNTY5MDI=" + } +} diff --git a/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-renamed-event.json b/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-renamed-event.json new file mode 100644 index 0000000000..d0636850c3 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-renamed-event.json @@ -0,0 +1,166 @@ +{ + "action": "renamed", + "changes": { + "repository": { + "name": { + "from": "foo" + } + } + }, + "repository": { + "id": 125428, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjU0Mjg=", + "name": "example-repo", + "full_name": "example-owner/example-repo", + "private": false, + "owner": { + "name": "example-owner", + "email": null, + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/example-owner", + "html_url": "https://ghe.example.net/example-owner", + "followers_url": "https://ghe.example.net/api/v3/users/example-owner/followers", + "following_url": "https://ghe.example.net/api/v3/users/example-owner/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/example-owner/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/example-owner/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/example-owner/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/example-owner/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/users/example-owner/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/example-owner/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://ghe.example.net/example-owner/example-repo", + "description": null, + "fork": false, + "url": "https://ghe.example.net/example-owner/example-repo", + "forks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/forks", + "keys_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/keys{/key_id}", + "collaborators_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/collaborators{/collaborator}", + "teams_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/teams", + "hooks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/hooks", + "issue_events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/events{/number}", + "events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/events", + "assignees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/assignees{/user}", + "branches_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/branches{/branch}", + "tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/tags", + "blobs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/blobs{/sha}", + "git_tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/tags{/sha}", + "git_refs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/refs{/sha}", + "trees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/trees{/sha}", + "statuses_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/statuses/{sha}", + "languages_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/languages", + "stargazers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/stargazers", + "contributors_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contributors", + "subscribers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscribers", + "subscription_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscription", + "commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/commits{/sha}", + "git_commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/commits{/sha}", + "comments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/comments{/number}", + "issue_comment_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/comments{/number}", + "contents_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contents/{+path}", + "compare_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/compare/{base}...{head}", + "merges_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/merges", + "archive_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/{archive_format}{/ref}", + "downloads_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/downloads", + "issues_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues{/number}", + "pulls_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/pulls{/number}", + "milestones_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/milestones{/number}", + "notifications_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/notifications{?since,all,participating}", + "labels_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/labels{/name}", + "releases_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/releases{/id}", + "deployments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/deployments", + "created_at": 1583321868, + "updated_at": "2022-09-08T05:09:21Z", + "pushed_at": 1741769053, + "git_url": "git://ghe.example.net/example-owner/example-repo.git", + "ssh_url": "git@ghe.example.net:example-owner/example-repo.git", + "clone_url": "https://ghe.example.net/example-owner/example-repo.git", + "svn_url": "https://ghe.example.net/example-owner/example-repo", + "homepage": null, + "size": 38734, + "stargazers_count": 0, + "watchers_count": 0, + "language": "TypeScript", + "has_issues": true, + "has_projects": true, + "has_downloads": false, + "has_wiki": false, + "has_pages": false, + "has_discussions": false, + "forks_count": 2, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 8, + "license": null, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 2, + "open_issues": 8, + "watchers": 0, + "default_branch": "master", + "stargazers": 0, + "master_branch": "master", + "organization": "example-owner", + "custom_properties": {} + }, + "organization": { + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "url": "https://ghe.example.net/api/v3/orgs/example-owner", + "repos_url": "https://ghe.example.net/api/v3/orgs/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/orgs/example-owner/events", + "hooks_url": "https://ghe.example.net/api/v3/orgs/example-owner/hooks", + "issues_url": "https://ghe.example.net/api/v3/orgs/example-owner/issues", + "members_url": "https://ghe.example.net/api/v3/orgs/example-owner/members{/member}", + "public_members_url": "https://ghe.example.net/api/v3/orgs/example-owner/public_members{/member}", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "description": null + }, + "enterprise": { + "id": 1, + "slug": "example", + "name": "Example", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://ghe.example.net/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://ghe.example.net/enterprises/example", + "created_at": "2019-03-30T15:03:21Z", + "updated_at": "2024-12-09T09:09:26Z" + }, + "sender": { + "login": "john", + "id": 11272, + "node_id": "MDQ6VXNlcjExMjcy", + "avatar_url": "https://ghe.example.net/avatars/u/11272?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/john", + "html_url": "https://ghe.example.net/john", + "followers_url": "https://ghe.example.net/api/v3/users/john/followers", + "following_url": "https://ghe.example.net/api/v3/users/john/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/john/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/john/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/john/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/john/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/john/repos", + "events_url": "https://ghe.example.net/api/v3/users/john/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/john/received_events", + "type": "User", + "site_admin": false + }, + "installation": { + "id": 56902, + "node_id": "MDIzOkludGVncmF0aW9uSW5zdGFsbGF0aW9uNTY5MDI=" + } +} diff --git a/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-transferred-event.json b/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-transferred-event.json new file mode 100644 index 0000000000..eb852dc6db --- /dev/null +++ b/plugins/catalog-backend-module-github/src/events/__fixtures__/repository-transferred-event.json @@ -0,0 +1,179 @@ +{ + "action": "transferred", + "changes": { + "owner": { + "from": { + "organization": { + "login": "foo", + "id": 6180, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "url": "https://ghe.example.net/api/v3/orgs/foo", + "repos_url": "https://ghe.example.net/api/v3/orgs/foo/repos", + "events_url": "https://ghe.example.net/api/v3/orgs/foo/events", + "hooks_url": "https://ghe.example.net/api/v3/orgs/foo/hooks", + "issues_url": "https://ghe.example.net/api/v3/orgs/foo/issues", + "members_url": "https://ghe.example.net/api/v3/orgs/foo/members{/member}", + "public_members_url": "https://ghe.example.net/api/v3/orgs/foo/public_members{/member}", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "description": null + } + } + } + }, + "repository": { + "id": 125428, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjU0Mjg=", + "name": "example-repo", + "full_name": "example-owner/example-repo", + "private": false, + "owner": { + "name": "example-owner", + "email": null, + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/example-owner", + "html_url": "https://ghe.example.net/example-owner", + "followers_url": "https://ghe.example.net/api/v3/users/example-owner/followers", + "following_url": "https://ghe.example.net/api/v3/users/example-owner/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/example-owner/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/example-owner/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/example-owner/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/example-owner/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/users/example-owner/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/example-owner/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://ghe.example.net/example-owner/example-repo", + "description": null, + "fork": false, + "url": "https://ghe.example.net/example-owner/example-repo", + "forks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/forks", + "keys_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/keys{/key_id}", + "collaborators_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/collaborators{/collaborator}", + "teams_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/teams", + "hooks_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/hooks", + "issue_events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/events{/number}", + "events_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/events", + "assignees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/assignees{/user}", + "branches_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/branches{/branch}", + "tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/tags", + "blobs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/blobs{/sha}", + "git_tags_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/tags{/sha}", + "git_refs_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/refs{/sha}", + "trees_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/trees{/sha}", + "statuses_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/statuses/{sha}", + "languages_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/languages", + "stargazers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/stargazers", + "contributors_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contributors", + "subscribers_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscribers", + "subscription_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/subscription", + "commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/commits{/sha}", + "git_commits_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/git/commits{/sha}", + "comments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/comments{/number}", + "issue_comment_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues/comments{/number}", + "contents_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/contents/{+path}", + "compare_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/compare/{base}...{head}", + "merges_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/merges", + "archive_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/{archive_format}{/ref}", + "downloads_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/downloads", + "issues_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/issues{/number}", + "pulls_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/pulls{/number}", + "milestones_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/milestones{/number}", + "notifications_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/notifications{?since,all,participating}", + "labels_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/labels{/name}", + "releases_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/releases{/id}", + "deployments_url": "https://ghe.example.net/api/v3/repos/example-owner/example-repo/deployments", + "created_at": 1583321868, + "updated_at": "2022-09-08T05:09:21Z", + "pushed_at": 1741769053, + "git_url": "git://ghe.example.net/example-owner/example-repo.git", + "ssh_url": "git@ghe.example.net:example-owner/example-repo.git", + "clone_url": "https://ghe.example.net/example-owner/example-repo.git", + "svn_url": "https://ghe.example.net/example-owner/example-repo", + "homepage": null, + "size": 38734, + "stargazers_count": 0, + "watchers_count": 0, + "language": "TypeScript", + "has_issues": true, + "has_projects": true, + "has_downloads": false, + "has_wiki": false, + "has_pages": false, + "has_discussions": false, + "forks_count": 2, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 8, + "license": null, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 2, + "open_issues": 8, + "watchers": 0, + "default_branch": "master", + "stargazers": 0, + "master_branch": "master", + "organization": "example-owner", + "custom_properties": {} + }, + "organization": { + "login": "example-owner", + "id": 6181, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjYxODE=", + "url": "https://ghe.example.net/api/v3/orgs/example-owner", + "repos_url": "https://ghe.example.net/api/v3/orgs/example-owner/repos", + "events_url": "https://ghe.example.net/api/v3/orgs/example-owner/events", + "hooks_url": "https://ghe.example.net/api/v3/orgs/example-owner/hooks", + "issues_url": "https://ghe.example.net/api/v3/orgs/example-owner/issues", + "members_url": "https://ghe.example.net/api/v3/orgs/example-owner/members{/member}", + "public_members_url": "https://ghe.example.net/api/v3/orgs/example-owner/public_members{/member}", + "avatar_url": "https://ghe.example.net/avatars/u/6181?", + "description": null + }, + "enterprise": { + "id": 1, + "slug": "example", + "name": "Example", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://ghe.example.net/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://ghe.example.net/enterprises/example", + "created_at": "2019-03-30T15:03:21Z", + "updated_at": "2024-12-09T09:09:26Z" + }, + "sender": { + "login": "john", + "id": 11272, + "node_id": "MDQ6VXNlcjExMjcy", + "avatar_url": "https://ghe.example.net/avatars/u/11272?", + "gravatar_id": "", + "url": "https://ghe.example.net/api/v3/users/john", + "html_url": "https://ghe.example.net/john", + "followers_url": "https://ghe.example.net/api/v3/users/john/followers", + "following_url": "https://ghe.example.net/api/v3/users/john/following{/other_user}", + "gists_url": "https://ghe.example.net/api/v3/users/john/gists{/gist_id}", + "starred_url": "https://ghe.example.net/api/v3/users/john/starred{/owner}{/repo}", + "subscriptions_url": "https://ghe.example.net/api/v3/users/john/subscriptions", + "organizations_url": "https://ghe.example.net/api/v3/users/john/orgs", + "repos_url": "https://ghe.example.net/api/v3/users/john/repos", + "events_url": "https://ghe.example.net/api/v3/users/john/events{/privacy}", + "received_events_url": "https://ghe.example.net/api/v3/users/john/received_events", + "type": "User", + "site_admin": false + }, + "installation": { + "id": 56902, + "node_id": "MDIzOkludGVncmF0aW9uSW5zdGFsbGF0aW9uNTY5MDI=" + } +} diff --git a/plugins/catalog-backend-module-github/src/events/analyzeGithubWebhookEvent.test.ts b/plugins/catalog-backend-module-github/src/events/analyzeGithubWebhookEvent.test.ts new file mode 100644 index 0000000000..86e2b58768 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/events/analyzeGithubWebhookEvent.test.ts @@ -0,0 +1,737 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mockServices } from '@backstage/backend-test-utils'; +import { analyzeGithubWebhookEvent } from './analyzeGithubWebhookEvent'; +import { OctokitProviderService } from '../util/octokitProviderService'; + +const isRelevantPath = (path: string): boolean => path.endsWith('.yaml'); + +describe('analyzeGithubWebhookEvent', () => { + const octokit = { + rest: { + repos: { + getCommit: jest.fn(), + }, + }, + }; + const octokitProvider = { + getOctokit: jest.fn(async () => octokit as any), + } satisfies OctokitProviderService; + const logger = mockServices.logger.mock(); + + function mockGetCommit( + first: { filename: string; status: string; previous_filename?: string }, + second?: { filename: string; status: string; previous_filename?: string }, + ) { + octokit.rest.repos.getCommit.mockImplementation( + async (params: { ref: string }) => { + if (params.ref === '8e0f23d8853333840f763bd7a87aa17aa6ed4b3d') { + return { + data: { + html_url: + 'https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d', + files: [first], + }, + }; + } else if (params.ref === '677c191848d92449c3d978becc71a59ea3e9777e') { + return { + data: { + html_url: + 'https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e', + files: second ? [second] : [], + }, + }; + } + throw new Error(`Unexpected commit ref: ${params.ref}`); + }, + ); + } + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('push', () => { + it('refreshes on changes, unregisters on deletes, for the simple case', async () => { + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-simple.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.updated", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/catalog-info.yaml", + }, + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + }, + "type": "location.deleted", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/catalog-info-2.yaml", + }, + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + }, + "type": "location.deleted", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/catalog-info-3.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - added', async () => { + mockGetCommit({ filename: 'a.yaml', status: 'added' }); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.created", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - added then added', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'added' }, + { filename: 'a.yaml', status: 'added' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.created", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - added then removed', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'added' }, + { filename: 'a.yaml', status: 'removed' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [], + "result": "ok", + } + `); + }); + + it('handles push - added then changed', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'added' }, + { filename: 'a.yaml', status: 'changed' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.created", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - added then renamed', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'added' }, + { filename: 'b.yaml', status: 'renamed', previous_filename: 'a.yaml' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + }, + "type": "location.created", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/b.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - removed', async () => { + mockGetCommit({ filename: 'a.yaml', status: 'removed' }); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.deleted", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - removed then added', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'removed' }, + { filename: 'a.yaml', status: 'added' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + }, + "type": "location.updated", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - removed then removed', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'removed' }, + { filename: 'a.yaml', status: 'removed' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.deleted", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - removed then changed', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'removed' }, + { filename: 'a.yaml', status: 'changed' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.deleted", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('ignores push - removed then renamed', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'removed' }, + { filename: 'b.yaml', status: 'renamed', previous_filename: 'a.yaml' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [], + "result": "ok", + } + `); + }); + + it('handles push - changed', async () => { + mockGetCommit({ filename: 'a.yaml', status: 'changed' }); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.updated", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - changed then added', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'changed' }, + { filename: 'a.yaml', status: 'added' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.updated", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - changed then removed', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'changed' }, + { filename: 'a.yaml', status: 'removed' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + }, + "type": "location.deleted", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - changed then changed', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'changed' }, + { filename: 'a.yaml', status: 'changed' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "type": "location.updated", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - changed then renamed', async () => { + mockGetCommit( + { filename: 'a.yaml', status: 'changed' }, + { filename: 'b.yaml', status: 'renamed', previous_filename: 'a.yaml' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + }, + "fromUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + "toUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/b.yaml", + "type": "location.moved", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - renamed', async () => { + mockGetCommit({ + filename: 'b.yaml', + status: 'renamed', + previous_filename: 'a.yaml', + }); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "fromUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + "toUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/b.yaml", + "type": "location.moved", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - renamed then added', async () => { + mockGetCommit( + { filename: 'b.yaml', status: 'renamed', previous_filename: 'a.yaml' }, + { filename: 'b.yaml', status: 'added' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "fromUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + "toUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/b.yaml", + "type": "location.moved", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - renamed then removed', async () => { + mockGetCommit( + { filename: 'b.yaml', status: 'renamed', previous_filename: 'a.yaml' }, + { filename: 'b.yaml', status: 'removed' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + }, + "type": "location.deleted", + "url": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - renamed then changed', async () => { + mockGetCommit( + { filename: 'b.yaml', status: 'renamed', previous_filename: 'a.yaml' }, + { filename: 'b.yaml', status: 'changed' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/8e0f23d8853333840f763bd7a87aa17aa6ed4b3d", + }, + "fromUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + "toUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/b.yaml", + "type": "location.moved", + }, + ], + "result": "ok", + } + `); + }); + + it('handles push - renamed then renamed', async () => { + mockGetCommit( + { filename: 'b.yaml', status: 'renamed', previous_filename: 'a.yaml' }, + { filename: 'c.yaml', status: 'renamed', previous_filename: 'b.yaml' }, + ); + + await expect( + analyzeGithubWebhookEvent( + 'push', + require('./__fixtures__/push-event-complex.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "context": { + "commitUrl": "https://ghe.example.net/example-owner/example-repo/commit/677c191848d92449c3d978becc71a59ea3e9777e", + }, + "fromUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/a.yaml", + "toUrl": "https://ghe.example.net/example-owner/example-repo/blob/master/c.yaml", + "type": "location.moved", + }, + ], + "result": "ok", + } + `); + }); + }); + + describe('repository.deleted', () => { + it('unregisters', async () => { + await expect( + analyzeGithubWebhookEvent( + 'repository', + require('./__fixtures__/repository-deleted-event.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "type": "repository.deleted", + "url": "https://ghe.example.net/example-owner/example-repo", + }, + ], + "result": "ok", + } + `); + }); + }); + + describe('repository.archived', () => { + it('refreshes', async () => { + await expect( + analyzeGithubWebhookEvent( + 'repository', + require('./__fixtures__/repository-archived-event.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "type": "repository.updated", + "url": "https://ghe.example.net/example-owner/example-repo", + }, + ], + "result": "ok", + } + `); + }); + }); + + describe('repository.renamed', () => { + it('moves', async () => { + await expect( + analyzeGithubWebhookEvent( + 'repository', + require('./__fixtures__/repository-renamed-event.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "fromUrl": "https://ghe.example.net/example-owner/foo", + "toUrl": "https://ghe.example.net/example-owner/example-repo", + "type": "repository.moved", + }, + ], + "result": "ok", + } + `); + }); + }); + + describe('repository.transferred', () => { + it('moves', async () => { + await expect( + analyzeGithubWebhookEvent( + 'repository', + require('./__fixtures__/repository-transferred-event.json'), + { octokitProvider, logger, isRelevantPath }, + ), + ).resolves.toMatchInlineSnapshot(` + { + "events": [ + { + "fromUrl": "https://ghe.example.net/foo/example-repo", + "toUrl": "https://ghe.example.net/example-owner/example-repo", + "type": "repository.moved", + }, + ], + "result": "ok", + } + `); + }); + }); +}); diff --git a/plugins/catalog-backend-module-github/src/events/analyzeGithubWebhookEvent.ts b/plugins/catalog-backend-module-github/src/events/analyzeGithubWebhookEvent.ts new file mode 100644 index 0000000000..b997e041fa --- /dev/null +++ b/plugins/catalog-backend-module-github/src/events/analyzeGithubWebhookEvent.ts @@ -0,0 +1,659 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LoggerService } from '@backstage/backend-plugin-api'; +import { InputError } from '@backstage/errors'; +import { CatalogScmEvent } from '@backstage/plugin-catalog-node/alpha'; +import type { + Commit, + Organization, + PushEvent, + Repository, + RepositoryArchivedEvent, + RepositoryCreatedEvent, + RepositoryDeletedEvent, + RepositoryEditedEvent, + RepositoryEvent, + RepositoryPrivatizedEvent, + RepositoryPublicizedEvent, + RepositoryRenamedEvent, + RepositoryTransferredEvent, + RepositoryUnarchivedEvent, + User, +} from '@octokit/webhooks-types'; +import { OctokitProviderService } from '../util/octokitProviderService'; +import { Octokit } from 'octokit'; + +export interface AnalyzeWebhookEventOptions { + octokitProvider: OctokitProviderService; + logger: LoggerService; + isRelevantPath: (path: string) => boolean; +} + +export type AnalyzeWebhookEventResult = + | { + result: 'unsupported-event'; + event: string; + } + | { + result: 'ignored'; + reason: string; + } + | { + result: 'aborted'; + reason: string; + } + | { + result: 'ok'; + events: CatalogScmEvent[]; + }; + +// Either a shorthand commit from a webhook event, or the full commit returned +// by the client API +type AnyCommit = + | Commit + | Awaited>['data']; + +/** + * Keeps track of intermediate state of individual paths while processing push events. + */ +type PathState = + | { + type: 'added'; + commit: AnyCommit; + } + | { + type: 'removed'; + commit: AnyCommit; + } + | { + type: 'modified'; + commit: AnyCommit; + } + | { + type: 'renamed'; + fromPath: string; + commit: AnyCommit; + } + | { + type: 'changed'; + commit: AnyCommit; + }; + +function pathStateToCatalogScmEvent( + path: string, + event: PathState, + repository: Repository, +): CatalogScmEvent { + const toBlobUrl = (p: string) => + `${repository.html_url}/blob/${repository.default_branch}/${p}`; + + const context = { + commitUrl: + 'html_url' in event.commit ? event.commit.html_url : event.commit.url, + }; + + switch (event.type) { + case 'added': + return { + type: 'location.created', + url: toBlobUrl(path), + context, + }; + case 'removed': + return { + type: 'location.deleted', + url: toBlobUrl(path), + context, + }; + case 'modified': + return { + type: 'location.updated', + url: toBlobUrl(path), + context, + }; + case 'renamed': + return { + type: 'location.moved', + fromUrl: toBlobUrl(event.fromPath), + toUrl: toBlobUrl(path), + context, + }; + case 'changed': + return { + type: 'location.updated', + url: toBlobUrl(path), + context, + }; + default: + // @ts-expect-error Intentionally expected, to check for exhaustive checking of the types + throw new Error(`Unknown file event type: ${event.type}`); + } +} + +/** + * Analyzes a GitHub webhook event and returns details about actions that the + * event might require. + */ +export async function analyzeGithubWebhookEvent( + eventType: string, + eventPayload: unknown, + options: AnalyzeWebhookEventOptions, +): Promise { + if ( + !eventPayload || + typeof eventPayload !== 'object' || + Array.isArray(eventPayload) + ) { + throw new InputError('GitHub webhook event payload is not an object'); + } + + if (eventType === 'push') { + return await onPushEvent(eventPayload as PushEvent, options); + } + + if (eventType === 'repository') { + const repositoryEvent = eventPayload as RepositoryEvent; + const action = repositoryEvent.action; + if (action === 'created') { + return await onRepositoryCreatedEvent(repositoryEvent); + } else if (action === 'deleted') { + return await onRepositoryDeletedEvent(repositoryEvent); + } else if (action === 'archived') { + return await onRepositoryArchivedEvent(repositoryEvent); + } else if (action === 'unarchived') { + return await onRepositoryUnarchivedEvent(repositoryEvent); + } else if (action === 'privatized') { + return await onRepositoryPrivatizedEvent(repositoryEvent); + } else if (action === 'publicized') { + return await onRepositoryPublicizedEvent(repositoryEvent); + } else if (action === 'renamed') { + return await onRepositoryRenamedEvent(repositoryEvent); + } else if (action === 'edited') { + return await onRepositoryEditedEvent(repositoryEvent); + } else if (action === 'transferred') { + return await onRepositoryTransferredEvent(repositoryEvent); + } + return { + result: 'unsupported-event', + event: `${eventType}.${action}`, + }; + } + + return { + result: 'unsupported-event', + event: eventType, + }; +} + +// #region Push events + +async function onPushEvent( + event: PushEvent, + options: AnalyzeWebhookEventOptions, +): Promise { + const contextUrl = event.compare || ''; + + // NOTE: The following caveats are mentioned in https://docs.github.com/en/webhooks/webhook-events-and-payloads#push + // * Events will not be created if more than 5000 branches are pushed at once. + // * Events will not be created for tags when more than three tags are pushed at once. + // * The commits array includes a maximum of 2048 commits. If necessary, you can use the Commits API to fetch additional commits. + // TODO(freben): Do we need to support reading the commits using the separate API under some circumstances? + if (event.commits.length >= 2048) { + return { + result: 'aborted', + reason: `GitHub push event has too many commits (${event.commits.length}), assuming that this is not a complete push event: ${contextUrl}`, + }; + } + + // We ignore any event that doesn't target the default branch as this + // is where the metadata files should be stored for now. + const defaultBranchRef = `refs/heads/${event.repository.default_branch}`; + if (event.ref !== defaultBranchRef) { + return { + result: 'ignored', + reason: `GitHub push event did not target the default branch, found "${event.ref}" but expected "${defaultBranchRef}": ${contextUrl}`, + }; + } + + /* + * STEP 1: Find interesting commits + * + * We go through the commits and exclude the ones that are not interesting. We + * consider a commit interesting if it mentions any relevant paths. We don't + * want to unnecessarily call out to the GitHub API for commits that don't + * affect files that we care about. + * + * The raw webhook event only contains shorthand commit data, not the full set + * of information. Notably it does not contain the "files" array that details + * what precise type of action was taken on individual paths. Instead it + * presents three summary arrays per commit: "added", "removed", and + * "modified", that just hold path strings. + */ + + const interestingShorthandCommits = (event.commits ?? []).filter( + commit => + commit.added?.some(options.isRelevantPath) || + commit.removed?.some(options.isRelevantPath) || + commit.modified?.some(options.isRelevantPath), + ); + + if (!interestingShorthandCommits.length) { + return { + result: 'ignored', + reason: `GitHub push event did not affect any relevant paths: ${contextUrl}`, + }; + } + + /* + * STEP 2: Simple case + * + * We go through the interesting shorthand commits in time order and make a + * basic assessment of what happened. Most of the time this will be enough to + * determine that (for relevant paths) there were only additions or only + * removals - and possibly some changes. In that case we don't have to pay the + * cost of fetching the detailed commits, because it's safe to assume that + * there were no complex renames/moves or similar. + * + * If we find that any path is mentioned more than once, we bail out and go to + * the complex case below instead. + */ + + const hasAddedPaths = interestingShorthandCommits.some( + commit => (commit.added ?? []).filter(options.isRelevantPath).length > 0, + ); + const hasRemovedPaths = interestingShorthandCommits.some( + commit => (commit.removed ?? []).filter(options.isRelevantPath).length > 0, + ); + + if (!(hasAddedPaths && hasRemovedPaths)) { + const addedOrRemovedPaths = new Map(); + const changedPaths = new Map(); + + for (const commit of interestingShorthandCommits) { + for (const path of commit.modified?.filter(options.isRelevantPath) ?? + []) { + if (!addedOrRemovedPaths.has(path)) { + changedPaths.set(path, { type: 'changed', commit }); + } + } + for (const path of commit.added?.filter(options.isRelevantPath) ?? []) { + changedPaths.delete(path); + addedOrRemovedPaths.set(path, { type: 'added', commit }); + } + for (const path of commit.removed?.filter(options.isRelevantPath) ?? []) { + changedPaths.delete(path); + addedOrRemovedPaths.set(path, { type: 'removed', commit }); + } + } + + const allPaths = new Map([ + ...changedPaths.entries(), + ...addedOrRemovedPaths.entries(), + ]); + + return { + result: 'ok', + events: Array.from(allPaths.entries()).map(([path, e]) => + pathStateToCatalogScmEvent(path, e, event.repository), + ), + }; + } + + /* + * STEP 3: Complex case + * + * There seem to be a more complex overall set of changes here. There are both + * additions and removals that look interesting. Now we need to analyze the + * commits more carefully, to see which changes constitute purely individual + * additions/removals, and which ones are actually moves. We do this by + * iterating through the commits as fetched from the remote (since they + * contain richer information than the webhook), and computing the "compound" + * outcome (e.g. an add followed by a remove of the same file can be ignored). + */ + + const pathState = new Map(); + + const octokit = await options.octokitProvider.getOctokit( + event.repository.html_url, + ); + + for (const eventCommit of interestingShorthandCommits) { + // As noted in the getCommit documentation, if there's a large number of + // files in the commit then only at most 300 of them will be returned along + // with pagination link heasder, and then going up to a total of at most + // 3000 files. But we also want to use the convenient octokit API so we + // paginate in this kind of clunky way and end whenever there's no more rel + // next URL. + let commit: + | Awaited>['data'] + | undefined; + for (let page = 1; page <= 10; ++page) { + const response = await octokit.rest.repos.getCommit({ + page, + per_page: 300, + owner: event.repository.owner.login, + repo: event.repository.name, + ref: eventCommit.id, + }); + + if (!commit) { + commit = response.data; + } else { + commit.files = [ + ...(commit.files ?? []), + ...(response.data.files ?? []), + ]; + } + + if (!response.headers?.link?.includes('rel="next"')) { + break; + } + } + + if (!commit) { + throw new Error(`Failed to fetch commit for ${contextUrl}`); + } + + // We somewhat wastefully track all paths here whether they seem initially + // relevant or not, because we don't know yet whether they will be + // moved/renamed into or out of relevance. + for (const file of commit.files ?? []) { + const previous = pathState.get(file.previous_filename || file.filename); + let next: PathState | undefined; + if (file.status === 'added') { + if (!previous) { + // First time we see this file in this set of commits + next = { type: 'added', commit }; + } else if (previous.type === 'removed') { + // Removed and then added again - assume changes + next = { type: 'changed', commit }; + } else { + // Should not happen; added/changed/moved -> added + next = previous; + options.logger.debug( + `Unexpected commit state transition from ${previous.type} to ${file.status} in ${event.compare}`, + ); + } + } else if (file.status === 'removed') { + if (!previous) { + // First time we see this file in this set of commits + next = { type: 'removed', commit }; + } else if (previous.type === 'added') { + // It was first added and then removed - turn into noop + next = undefined; + } else if (previous.type === 'changed') { + // Changed and then removed; convert to removed + next = { type: 'removed', commit }; + } else if (previous.type === 'renamed') { + // It was renamed and then removed - convert to removal of the ORIGINAL path + if (!pathState.has(previous.fromPath)) { + pathState.set(previous.fromPath, { type: 'removed', commit }); + } + next = undefined; + } else { + // Should not happen; removed -> removed + next = previous; + options.logger.debug( + `Unexpected commit state transition from ${previous.type} to ${file.status} in ${event.compare}`, + ); + } + } else if (file.status === 'renamed') { + pathState.delete(file.previous_filename!); + if (!previous) { + // First time we see this file in this set of commits + next = { type: 'renamed', fromPath: file.previous_filename!, commit }; + } else if (previous.type === 'added') { + // It was first added and then moved - this sums to still just an add but in the new path + next = { type: 'added', commit }; + } else if (previous.type === 'changed') { + // Changed and then moved; convert to moved + next = { type: 'renamed', fromPath: file.previous_filename!, commit }; + } else if (previous.type === 'renamed') { + // It was renamed and then renamed again + next = { type: 'renamed', fromPath: previous.fromPath, commit }; + } else { + // Should not happen; removed -> renamed + next = undefined; + options.logger.debug( + `Unexpected commit state transition from ${previous.type} to ${file.status} in ${event.compare}`, + ); + } + } else if (file.status === 'changed' || file.status === 'modified') { + if (!previous) { + // First time we see this file in this set of commits + next = { type: 'changed', commit }; + } else if (previous.type === 'added') { + // It was first added and then changed - still an add + next = previous; + } else if (previous.type === 'changed') { + // Changed twice - still just a change + next = previous; + } else if (previous.type === 'renamed') { + // Renamed and then changed - still a rename + next = previous; + } else { + // Should not happen; removed -> changed but still keep the old state because it makes the most sense + next = previous; + options.logger.debug( + `Unexpected commit state transition from ${previous.type} to ${file.status} in ${event.compare}`, + ); + } + } else { + // remaining statuses are 'copied' and 'unchanged' - ignoring + next = previous; + } + + if (next) { + pathState.set(file.filename, next); + } else { + pathState.delete(file.filename); + } + } + } + + return { + result: 'ok', + events: Array.from(pathState.entries()).flatMap(([path, e]) => + options.isRelevantPath(path) + ? [pathStateToCatalogScmEvent(path, e, event.repository)] + : [], + ), + }; +} + +// #endregion +// #region Repository events + +async function onRepositoryArchivedEvent( + event: RepositoryArchivedEvent, +): Promise { + return { + result: 'ok', + events: [ + { + type: 'repository.updated', + url: event.repository.html_url, + }, + ], + }; +} + +async function onRepositoryUnarchivedEvent( + event: RepositoryUnarchivedEvent, +): Promise { + return { + result: 'ok', + events: [ + { + type: 'repository.updated', + url: event.repository.html_url, + }, + ], + }; +} + +async function onRepositoryPrivatizedEvent( + event: RepositoryPrivatizedEvent, +): Promise { + return { + result: 'ok', + events: [ + { + type: 'repository.updated', + url: event.repository.html_url, + }, + ], + }; +} + +async function onRepositoryPublicizedEvent( + event: RepositoryPublicizedEvent, +): Promise { + return { + result: 'ok', + events: [ + { + type: 'repository.updated', + url: event.repository.html_url, + }, + ], + }; +} + +async function onRepositoryCreatedEvent( + event: RepositoryCreatedEvent, +): Promise { + return { + result: 'ok', + events: [ + { + type: 'repository.created', + url: event.repository.html_url, + }, + ], + }; +} + +async function onRepositoryDeletedEvent( + event: RepositoryDeletedEvent, +): Promise { + return { + result: 'ok', + events: [ + { + type: 'repository.deleted', + url: event.repository.html_url, + }, + ], + }; +} + +async function onRepositoryRenamedEvent( + event: RepositoryRenamedEvent, +): Promise { + const oldUrlPrefix = `${event.repository.owner.html_url}/${event.changes.repository.name.from}`; + const newUrlPrefix = `${event.repository.html_url}`; + return { + result: 'ok', + events: [ + { + type: 'repository.moved', + fromUrl: oldUrlPrefix, + toUrl: newUrlPrefix, + }, + ], + }; +} + +async function onRepositoryEditedEvent( + event: RepositoryEditedEvent, +): Promise { + if (event.changes.default_branch) { + const oldUrlPrefix = `${event.repository.html_url}/blob/${event.changes.default_branch.from}`; + const newUrlPrefix = `${event.repository.html_url}/blob/${event.repository.default_branch}`; + return { + result: 'ok', + events: [ + { + type: 'repository.moved', + fromUrl: oldUrlPrefix, + toUrl: newUrlPrefix, + }, + ], + }; + } + + const changes = Object.keys(event.changes) + .map(c => `'${c}'`) + .join(', '); + return { + result: 'ignored', + reason: `GitHub repository edited event not handled: [${changes}]`, + }; +} + +async function onRepositoryTransferredEvent( + event: RepositoryTransferredEvent, +): Promise { + // TODO(freben): The web interface allows renaming while transferring, but + // we do not yet have a way of detecting that because weirdly, the actual + // webhooks that then get sent are + // + // - First a transferred event OLD_OWNER -> NEW_OWNER/NEW_REPO_NAME + // (no mention of the OLD_REPO_NAME anywhere in the event payload) + // + // - Then a renamed event for NEW_OWNER of OLD_REPO_NAME -> NEW_REPO_NAME + // (where the former of course never existed as far as we know; + // no mention of the OLD_OWNER anywhere in the event payload) + // + // Handling this seems to need some form of state handling across events. + // + // Therefore, this code expects that OLD_REPO_NAME === NEW_REPO_NAME + // and assumes that we did not perform a rename as part of the transfer. + + // The documentation states that there is an organization property in from, + // but the types disagree. Hence the cast. + // https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=transferred#repository + const from = event.changes.owner.from as { + user?: User; + organization?: Organization; + }; + + let oldOwnerUrl = from.user?.html_url ?? from.organization?.html_url; + if (!oldOwnerUrl) { + // Some servers do not supply an html_url field, so we + // construct it by removing the last two segments of the + // target URL instead (which we know ends with "/owner/repo") + const base = event.repository.html_url.split('/').slice(0, -2).join('/'); + oldOwnerUrl = `${base}/${from.user?.login ?? from.organization?.login}`; + } + + const oldUrlPrefix = `${oldOwnerUrl}/${event.repository.name}`; + const newUrlPrefix = `${event.repository.html_url}`; + return { + result: 'ok', + events: [ + { + type: 'repository.moved', + fromUrl: oldUrlPrefix, + toUrl: newUrlPrefix, + }, + ], + }; +} + +// #endregion diff --git a/plugins/catalog-backend-module-github/src/module/githubCatalogModule.ts b/plugins/catalog-backend-module-github/src/module/githubCatalogModule.ts index d09b2c2066..6177b88697 100644 --- a/plugins/catalog-backend-module-github/src/module/githubCatalogModule.ts +++ b/plugins/catalog-backend-module-github/src/module/githubCatalogModule.ts @@ -23,12 +23,15 @@ import { catalogProcessingExtensionPoint, } from '@backstage/plugin-catalog-node'; import { catalogServiceRef } from '@backstage/plugin-catalog-node'; +import { catalogScmEventsServiceRef } from '@backstage/plugin-catalog-node/alpha'; import { eventsServiceRef } from '@backstage/plugin-events-node'; import { GithubEntityProvider } from '../providers/GithubEntityProvider'; import { GithubLocationAnalyzer } from '../analyzers/GithubLocationAnalyzer'; +import { octokitProviderServiceRef } from '../util/octokitProviderService'; +import { GithubScmEventsBridge } from '../events/GithubScmEventsBridge'; /** - * Registers the `GithubEntityProvider` with the catalog processing extension point. + * Registers all relevant GitHub integration points into the catalog backend. * * @public */ @@ -46,6 +49,9 @@ export const githubCatalogModule = createBackendModule({ logger: coreServices.logger, scheduler: coreServices.scheduler, catalog: catalogServiceRef, + octokitProvider: octokitProviderServiceRef, + catalogScmEvents: catalogScmEventsServiceRef, + lifecycle: coreServices.lifecycle, }, async init({ catalogProcessing, @@ -56,6 +62,9 @@ export const githubCatalogModule = createBackendModule({ catalogAnalyzers, auth, catalog, + octokitProvider, + catalogScmEvents, + lifecycle, }) { catalogAnalyzers.addScmLocationAnalyzer( new GithubLocationAnalyzer({ @@ -72,6 +81,19 @@ export const githubCatalogModule = createBackendModule({ scheduler, }), ); + + const bridge = new GithubScmEventsBridge({ + logger, + events, + octokitProvider, + catalogScmEvents, + }); + lifecycle.addStartupHook(async () => { + await bridge.start(); + }); + lifecycle.addShutdownHook(async () => { + await bridge.stop(); + }); }, }); }, diff --git a/plugins/catalog-backend-module-github/src/util/octokitProviderService.ts b/plugins/catalog-backend-module-github/src/util/octokitProviderService.ts new file mode 100644 index 0000000000..3134c5eeb3 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/util/octokitProviderService.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + coreServices, + createServiceFactory, + createServiceRef, + RootConfigService, +} from '@backstage/backend-plugin-api'; +import { + DefaultGithubCredentialsProvider, + GithubCredentialsProvider, + ScmIntegrationRegistry, + ScmIntegrations, +} from '@backstage/integration'; +import { durationToMilliseconds, HumanDuration } from '@backstage/types'; +import { Octokit } from 'octokit'; + +export interface OctokitProviderService { + getOctokit: (url: string) => Promise; +} + +class OctokitProviderImpl implements OctokitProviderService { + readonly #integrations: ScmIntegrationRegistry; + readonly #githubCredentials: GithubCredentialsProvider; + readonly #octokitCache: Map; + readonly #octokitCacheTtl: HumanDuration; + + constructor(config: RootConfigService) { + this.#integrations = ScmIntegrations.fromConfig(config); + this.#githubCredentials = DefaultGithubCredentialsProvider.fromIntegrations( + this.#integrations, + ); + this.#octokitCache = new Map(); + this.#octokitCacheTtl = { hours: 1 }; + } + + async getOctokit(url: string): Promise { + // TODO(freben): Be smart and cache these more granularly, e.g. by + // organization or even repo. + const integration = this.#integrations.github.byUrl(url); + if (!integration) { + throw new Error(`No integration found for url: ${url}`); + } + const key = integration.config.host; + + if (this.#octokitCache.has(key)) { + return this.#octokitCache.get(key)!; + } + + const { createCallbackAuth } = await import('@octokit/auth-callback'); + + const octokit = new Octokit({ + baseUrl: integration.config.apiBaseUrl, + authStrategy: createCallbackAuth, + auth: { + callback: async () => { + try { + const credentials = await this.#githubCredentials.getCredentials({ + url, + }); + return credentials.token; + } catch { + return undefined; + } + }, + }, + }); + + this.#octokitCache.set(key, octokit); + setTimeout(() => { + this.#octokitCache.delete(key); + }, durationToMilliseconds(this.#octokitCacheTtl)); + + return octokit; + } +} + +/** + * This will have to live here, until we have a proper shared one in an + * integrations layer. + */ +export const octokitProviderServiceRef = + createServiceRef({ + id: 'octokitProvider', + scope: 'root', + defaultFactory: async service => + createServiceFactory({ + service, + deps: { config: coreServices.rootConfig }, + async factory({ config }) { + return new OctokitProviderImpl(config); + }, + }), + }); diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts index 05e078723d..a2ffdee053 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts @@ -25,17 +25,34 @@ import { DbSearchRow, } from '../database/tables'; import { DefaultLocationStore } from './DefaultLocationStore'; +import { CatalogScmEventsServiceSubscriber } from '@backstage/plugin-catalog-node/alpha'; +import waitFor from 'wait-for-expect'; jest.setTimeout(60_000); describe('DefaultLocationStore', () => { const databases = TestDatabases.create(); + const mockScmEvents = { + subscribe: jest.fn(), + publish: jest.fn(), + }; + let subscriber: CatalogScmEventsServiceSubscriber | undefined; + + beforeEach(() => { + jest.clearAllMocks(); + + subscriber = undefined; + mockScmEvents.subscribe.mockImplementation(sub => { + subscriber = sub; + return { unsubscribe: () => {} }; + }); + }); async function createLocationStore(databaseId: TestDatabaseId) { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); const connection = { applyMutation: jest.fn(), refresh: jest.fn() }; - const store = new DefaultLocationStore(knex); + const store = new DefaultLocationStore(knex, mockScmEvents); await store.connect(connection); return { store, connection, knex }; } @@ -236,4 +253,460 @@ describe('DefaultLocationStore', () => { }, ); }); + + describe('SCM event handling', () => { + describe.each(databases.eachSupportedId())('%p', databaseId => { + it('handles location.deleted', async () => { + const { store, knex, connection } = await createLocationStore( + databaseId, + ); + expect(subscriber).not.toBeUndefined(); + + // Prepare + + const matchTarget = + 'https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml'; + const otherTarget = + 'https://github.com/backstage/other/blob/master/folder/catalog-info.yaml'; + + await store.createLocation({ + type: 'url', + target: matchTarget, + }); + await store.createLocation({ + type: 'url', + target: otherTarget, + }); + + await waitFor(async () => { + await expect( + knex('locations') + .where('type', 'url') + .orderBy('target', 'asc'), + ).resolves.toEqual([ + { + id: expect.any(String), + type: 'url', + target: matchTarget, + }, + { + id: expect.any(String), + type: 'url', + target: otherTarget, + }, + ]); + }); + + await waitFor(async () => { + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: matchTarget, + type: 'url', + }, + }), + locationKey: `url:${matchTarget}`, + }, + ], + removed: [], + }); + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: otherTarget, + type: 'url', + }, + }), + locationKey: `url:${otherTarget}`, + }, + ], + removed: [], + }); + }); + + // Act + + await subscriber!.onEvents([ + { type: 'location.deleted', url: matchTarget }, + ]); + + // Verify + + await waitFor(async () => { + await expect( + knex('locations') + .where('type', 'url') + .orderBy('target', 'asc'), + ).resolves.toEqual([ + { id: expect.any(String), type: 'url', target: otherTarget }, + ]); + + expect(connection.applyMutation).toHaveBeenLastCalledWith({ + type: 'delta', + added: [], + removed: [ + { + entity: expect.objectContaining({ + spec: { target: matchTarget, type: 'url' }, + }), + }, + ], + }); + }); + }); + + it('handles location.moved', async () => { + const { store, knex, connection } = await createLocationStore( + databaseId, + ); + expect(subscriber).not.toBeUndefined(); + + // Prepare + + const matchTarget = + 'https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml'; + const otherTarget = + 'https://github.com/backstage/other/blob/master/folder/catalog-info.yaml'; + + await store.createLocation({ + type: 'url', + target: matchTarget, + }); + await store.createLocation({ + type: 'url', + target: otherTarget, + }); + + await waitFor(async () => { + await expect( + knex('locations') + .where('type', 'url') + .orderBy('target', 'asc'), + ).resolves.toEqual([ + { + id: expect.any(String), + type: 'url', + target: matchTarget, + }, + { + id: expect.any(String), + type: 'url', + target: otherTarget, + }, + ]); + }); + + await waitFor(async () => { + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: matchTarget, + type: 'url', + }, + }), + locationKey: `url:${matchTarget}`, + }, + ], + removed: [], + }); + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: otherTarget, + type: 'url', + }, + }), + locationKey: `url:${otherTarget}`, + }, + ], + removed: [], + }); + }); + + // Act + + await subscriber!.onEvents([ + { + type: 'location.moved', + fromUrl: matchTarget, + toUrl: + 'https://github.com/backstage/freben/blob/master/catalog-info.yaml', + }, + ]); + + // Verify + + await waitFor(async () => { + await expect( + knex('locations') + .where('type', 'url') + .orderBy('target', 'asc'), + ).resolves.toEqual([ + { + id: expect.any(String), + type: 'url', + target: + 'https://github.com/backstage/freben/blob/master/catalog-info.yaml', + }, + { id: expect.any(String), type: 'url', target: otherTarget }, + ]); + + expect(connection.applyMutation).toHaveBeenLastCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: + 'https://github.com/backstage/freben/blob/master/catalog-info.yaml', + type: 'url', + }, + }), + locationKey: `url:https://github.com/backstage/freben/blob/master/catalog-info.yaml`, + }, + ], + removed: [], + }); + }); + }); + + it('handles repository.deleted', async () => { + const { store, knex, connection } = await createLocationStore( + databaseId, + ); + expect(subscriber).not.toBeUndefined(); + + // Prepare + + const matchPrefix = 'https://github.com/backstage/demo'; + const matchTarget = + 'https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml'; + const otherTarget = + 'https://github.com/backstage/other/blob/master/folder/catalog-info.yaml'; + + await store.createLocation({ + type: 'url', + target: matchTarget, + }); + await store.createLocation({ + type: 'url', + target: otherTarget, + }); + + await waitFor(async () => { + await expect( + knex('locations') + .where('type', 'url') + .orderBy('target', 'asc'), + ).resolves.toEqual([ + { + id: expect.any(String), + type: 'url', + target: matchTarget, + }, + { + id: expect.any(String), + type: 'url', + target: otherTarget, + }, + ]); + }); + + await waitFor(async () => { + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: matchTarget, + type: 'url', + }, + }), + locationKey: `url:${matchTarget}`, + }, + ], + removed: [], + }); + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: otherTarget, + type: 'url', + }, + }), + locationKey: `url:${otherTarget}`, + }, + ], + removed: [], + }); + }); + + // Act + + await subscriber!.onEvents([ + { type: 'repository.deleted', url: matchPrefix }, + ]); + + // Verify + + await waitFor(async () => { + await expect( + knex('locations') + .where('type', 'url') + .orderBy('target', 'asc'), + ).resolves.toEqual([ + { id: expect.any(String), type: 'url', target: otherTarget }, + ]); + + expect(connection.applyMutation).toHaveBeenLastCalledWith({ + type: 'delta', + added: [], + removed: [ + { + entity: expect.objectContaining({ + spec: { target: matchTarget, type: 'url' }, + }), + }, + ], + }); + }); + }); + + it('handles repository.moved', async () => { + const { store, knex, connection } = await createLocationStore( + databaseId, + ); + expect(subscriber).not.toBeUndefined(); + + // Prepare + + const matchTarget = + 'https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml'; + const otherTarget = + 'https://github.com/backstage/other/blob/master/folder/catalog-info.yaml'; + + await store.createLocation({ + type: 'url', + target: matchTarget, + }); + await store.createLocation({ + type: 'url', + target: otherTarget, + }); + + await waitFor(async () => { + await expect( + knex('locations') + .where('type', 'url') + .orderBy('target', 'asc'), + ).resolves.toEqual([ + { + id: expect.any(String), + type: 'url', + target: matchTarget, + }, + { + id: expect.any(String), + type: 'url', + target: otherTarget, + }, + ]); + }); + + await waitFor(async () => { + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: matchTarget, + type: 'url', + }, + }), + locationKey: `url:${matchTarget}`, + }, + ], + removed: [], + }); + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: otherTarget, + type: 'url', + }, + }), + locationKey: `url:${otherTarget}`, + }, + ], + removed: [], + }); + }); + + // Act + + await subscriber!.onEvents([ + { + type: 'repository.moved', + fromUrl: 'https://github.com/backstage/demo', + toUrl: 'https://github.com/freben/demo-renamed', + }, + ]); + + // Verify + + await waitFor(async () => { + await expect( + knex('locations') + .where('type', 'url') + .orderBy('target', 'asc'), + ).resolves.toEqual([ + { id: expect.any(String), type: 'url', target: otherTarget }, + { + id: expect.any(String), + type: 'url', + target: + 'https://github.com/freben/demo-renamed/blob/master/folder/catalog-info.yaml', + }, + ]); + + expect(connection.applyMutation).toHaveBeenLastCalledWith({ + type: 'delta', + added: [ + { + entity: expect.objectContaining({ + spec: { + target: + 'https://github.com/freben/demo-renamed/blob/master/folder/catalog-info.yaml', + type: 'url', + }, + }), + locationKey: `url:https://github.com/freben/demo-renamed/blob/master/folder/catalog-info.yaml`, + }, + ], + removed: [], + }); + }); + }); + }); + }); }); diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts index d7cb945213..6a8967f760 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts @@ -36,13 +36,21 @@ import { parseLocationRef, stringifyEntityRef, } from '@backstage/catalog-model'; +import { + CatalogScmEvent, + CatalogScmEventsService, +} from '@backstage/plugin-catalog-node/alpha'; +import { chunk, uniqBy } from 'lodash'; +import parseGitUrl, { type GitUrl } from 'git-url-parse'; export class DefaultLocationStore implements LocationStore, EntityProvider { private _connection: EntityProviderConnection | undefined; private readonly db: Knex; + private readonly scmEvents: CatalogScmEventsService; - constructor(db: Knex) { + constructor(db: Knex, scmEvents: CatalogScmEventsService) { this.db = db; + this.scmEvents = scmEvents; } getProviderName(): string { @@ -185,6 +193,8 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { type: 'full', entities, }); + + this.scmEvents.subscribe({ onEvents: this.#onScmEvents.bind(this) }); } private async locations(dbOrTx: Knex.Transaction | Knex = this.db) { @@ -201,4 +211,260 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { })) ); } + + // #region SCM event handling + + async #onScmEvents(events: CatalogScmEvent[]): Promise { + const exactLocationsToDelete = new Set(); + const locationPrefixesToDelete = new Set(); + const exactLocationsToCreate = new Set(); + const locationPrefixesToMove = new Map(); + + for (const event of events) { + if (event.type === 'location.deleted') { + exactLocationsToDelete.add(event.url); + } else if (event.type === 'location.moved') { + // Since Location entities are named after their target URL, these + // unfortunately have to be translated into deletion and creation + exactLocationsToDelete.add(event.fromUrl); + exactLocationsToCreate.add(event.toUrl); + } else if (event.type === 'repository.deleted') { + locationPrefixesToDelete.add(event.url); + } else if (event.type === 'repository.moved') { + // These also have to be handled with deletions and creations + locationPrefixesToMove.set(event.fromUrl, event.toUrl); + } + } + + if (exactLocationsToDelete.size > 0) { + await this.#deleteLocationsByExactUrl(exactLocationsToDelete); + } + if (locationPrefixesToDelete.size > 0) { + await this.#deleteLocationsByUrlPrefix(locationPrefixesToDelete); + } + if (exactLocationsToCreate.size > 0) { + await this.#createLocationsByExactUrl(exactLocationsToCreate); + } + if (locationPrefixesToMove.size > 0) { + await this.#moveLocationsByUrlPrefix(locationPrefixesToMove); + } + } + + async #createLocationsByExactUrl(urls: Iterable): Promise { + let count = 0; + + for (const batch of chunk(Array.from(urls), 100)) { + const existingUrls = await this.db('locations') + .where('type', '=', 'url') + .where('target', 'in', batch) + .select() + .then(rows => new Set(rows.map(row => row.target))); + + const newLocations = batch + .filter(url => !existingUrls.has(url)) + .map(url => ({ id: uuid(), type: 'url', target: url })); + + if (newLocations.length) { + await this.db('locations').insert(newLocations); + + await this.connection.applyMutation({ + type: 'delta', + added: newLocations.map(location => { + const entity = locationSpecToLocationEntity({ location }); + return { entity, locationKey: getEntityLocationRef(entity) }; + }), + removed: [], + }); + + count += newLocations.length; + } + } + + return count; + } + + async #deleteLocationsByExactUrl(urls: Iterable): Promise { + let count = 0; + + for (const batch of chunk(Array.from(urls), 100)) { + const rows = await this.db('locations') + .where('type', '=', 'url') + .where('target', 'in', batch) + .select(); + + if (rows.length) { + await this.db('locations') + .where( + 'id', + 'in', + rows.map(row => row.id), + ) + .delete(); + + await this.connection.applyMutation({ + type: 'delta', + added: [], + removed: rows.map(row => ({ + entity: locationSpecToLocationEntity({ location: row }), + })), + }); + + count += rows.length; + } + } + + return count; + } + + async #deleteLocationsByUrlPrefix(urls: Iterable): Promise { + const matches = await this.#findLocationsByPrefixOrExactMatch(urls); + if (matches.length) { + await this.#deleteLocations(matches.map(l => l.row)); + } + + return matches.length; + } + + async #moveLocationsByUrlPrefix( + urlPrefixes: Map, + ): Promise { + let count = 0; + + for (const [fromPrefix, toPrefix] of urlPrefixes) { + if (fromPrefix === toPrefix) { + continue; + } + + if (fromPrefix.match(/[?#]/) || toPrefix.match(/[?#]/)) { + // TODO(freben): We can't yet support complex URL locations where e.g. + // the path can be anywhere in the URL including in the query or hash + // part. The code below currently assumes that we can use simple + // substring operations. + continue; + } + + const matches = await this.#findLocationsByPrefixOrExactMatch([ + fromPrefix, + ]); + if (matches.length) { + await this.#deleteLocations(matches.map(m => m.row)); + + await this.#createLocationsByExactUrl( + matches.map(m => { + const remainder = m.row.target + .slice(fromPrefix.length) + .replace(/^\/+/, ''); + if (!remainder) { + return toPrefix; + } + return `${toPrefix.replace(/\/+$/, '')}/${remainder}`; + }), + ); + + count += matches.length; + } + } + + return count; + } + + async #deleteLocations(rows: DbLocationsRow[]): Promise { + // Delete the location table entries (in chunks so as not to overload the + // knex query builder) + for (const ids of chunk( + rows.map(l => l.id), + 100, + )) { + await this.db('locations').whereIn('id', ids).delete(); + } + + // Delete the corresponding Location kind entities (this is efficiently + // chunked internally in the catalog) + await this.connection.applyMutation({ + type: 'delta', + added: [], + removed: rows.map(l => ({ + entity: locationSpecToLocationEntity({ location: l }), + })), + }); + } + + /** + * Given a "base" URL prefix, find all locations that are for paths at or + * below it. + * + * For example, given a base URL prefix of + * "https://github.com/backstage/backstage/blob/master/plugins", it will match + * locations inside the plugins directory, and nowhere else. + */ + async #findLocationsByPrefixOrExactMatch( + urls: Iterable, + ): Promise> { + const result = new Array<{ row: DbLocationsRow; parsed: GitUrl }>(); + + for (const url of urls) { + let base: GitUrl; + try { + base = parseGitUrl(url); + } catch (error) { + throw new Error(`Invalid URL prefix, could not parse: ${url}`); + } + + if (!base.owner || !base.name) { + throw new Error( + `Invalid URL prefix, missing owner or repository: ${url}`, + ); + } + + const pathPrefix = + base.filepath === '' || base.filepath.endsWith('/') + ? base.filepath + : `${base.filepath}/`; + + const rows = await this.db('locations') + .where('type', '=', 'url') + // Initial rough pruning to not have to go through them all + .where('target', 'like', `%${base.owner}%`) + .where('target', 'like', `%${base.name}%`) + .select(); + + result.push( + ...rows.flatMap(row => { + try { + // We do this pretty explicit set of checks because we want to support + // providers that have a URL format where the path isn't necessarily at + // the end of the URL string (e.g. in the query part). Some of these may + // be empty strings etc, but that's fine as long as they parse to the + // same thing as above. + const candidate = parseGitUrl(row.target); + + if ( + candidate.protocol === base.protocol && + candidate.resource === base.resource && + candidate.port === base.port && + candidate.organization === base.organization && + candidate.owner === base.owner && + candidate.name === base.name && + // If the base has no ref (for example didn't have the "/blob/master" + // part and therefore targeted an entire repository) then we match any + // ref below that + (!base.ref || candidate.ref === base.ref) && + // Match both on exact equality and any subpath with a slash between + (candidate.filepath === base.filepath || + candidate.filepath.startsWith(pathPrefix)) + ) { + return [{ row, parsed: candidate }]; + } + return []; + } catch { + return []; + } + }), + ); + } + + return uniqBy(result, entry => entry.row.id); + } + + // #endregion } diff --git a/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts new file mode 100644 index 0000000000..7db594b8fe --- /dev/null +++ b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts @@ -0,0 +1,252 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, +} from '@backstage/catalog-model'; +import { CatalogScmEventsServiceSubscriber } from '@backstage/plugin-catalog-node/alpha'; +import { Knex } from 'knex'; +import { applyDatabaseMigrations } from '../database/migrations'; +import { + DbRefreshKeysRow, + DbRefreshStateRow, + DbSearchRow, +} from '../database/tables'; +import { GenericScmEventRefreshProvider } from './GenericScmEventRefreshProvider'; + +describe('GenericScmEventRefreshProvider', () => { + const databases = TestDatabases.create(); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + async function initialize(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + + let subscriber: CatalogScmEventsServiceSubscriber | undefined; + const scmEvents = { + subscribe: jest.fn().mockImplementation(sub => { + subscriber = sub; + return { unsubscribe: () => {} }; + }), + publish: jest.fn(), + }; + + const store = new GenericScmEventRefreshProvider(knex, scmEvents); + + const connection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await store.connect(connection); + + return { store, connection, knex, subscriber: subscriber!, scmEvents }; + } + + async function insertRefreshState(knex: Knex, id: string) { + await knex('refresh_state').insert({ + entity_id: id, + entity_ref: `k:ns/${id}`, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: new Date('1980-01-01T00:00:00Z'), + last_discovery_at: new Date('1980-01-01T00:00:00Z'), + result_hash: 'h', + }); + } + + describe.each(databases.eachSupportedId())('%p', databaseId => { + it('handles location.updated', async () => { + const { knex, subscriber } = await initialize(databaseId); + + // Prepare + await insertRefreshState(knex, '1'); + await insertRefreshState(knex, '2'); + await insertRefreshState(knex, '3'); + await insertRefreshState(knex, '4'); + await insertRefreshState(knex, '5'); + await insertRefreshState(knex, '6'); + + await knex('refresh_keys').insert([ + // match exact blob + { + entity_id: '1', + key: 'url:https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml', + }, + // match tree + { + entity_id: '2', + key: 'url:https://github.com/backstage/demo/tree/master/folder/catalog-info.yaml', + }, + // skip + { + entity_id: '3', + key: 'url:https://github.com/backstage/demo/tree/master/folder/catalog-info.yaml2', + }, + ]); + + await knex('search').insert([ + // match exact blob in location + { + entity_id: '4', + key: `metadata.annotations.${ANNOTATION_LOCATION}`, + original_value: 'ignored', + value: + 'url:https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml', + }, + // match exact blob in origin location + { + entity_id: '5', + key: `metadata.annotations.${ANNOTATION_ORIGIN_LOCATION}`, + original_value: 'ignored', + value: + 'url:https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml', + }, + // skip + { + entity_id: '6', + key: `some-other-key`, + original_value: 'ignored', + value: + 'url:https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml', + }, + ]); + + // Act + await subscriber!.onEvents([ + { + type: 'location.updated', + url: 'https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml', + }, + ]); + + // Verify + await expect( + knex('refresh_state') + .select('entity_id') + .where('next_update_at', '>', new Date('1981-01-01T00:00:00Z')) + .orderBy('entity_id', 'asc'), + ).resolves.toEqual([ + { entity_id: '1' }, + { entity_id: '2' }, + { entity_id: '4' }, + { entity_id: '5' }, + ]); + }); + + it('handles repository.updated', async () => { + const { knex, subscriber } = await initialize(databaseId); + + await insertRefreshState(knex, '1'); + await insertRefreshState(knex, '2'); + await insertRefreshState(knex, '3'); + await insertRefreshState(knex, '4'); + await insertRefreshState(knex, '5'); + await insertRefreshState(knex, '6'); + await insertRefreshState(knex, '7'); + await insertRefreshState(knex, '8'); + + await knex('refresh_keys').insert([ + // match blob + { + entity_id: '1', + key: 'url:https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml', + }, + // match tree + { + entity_id: '2', + key: 'url:https://github.com/backstage/demo/tree/master/folder/catalog-info.yaml', + }, + // skip different but similar repo + { + entity_id: '3', + key: 'url:https://github.com/backstage/demo2/tree/master/folder/catalog-info.yaml', + }, + ]); + + await knex('search').insert([ + // match blob in location + { + entity_id: '4', + key: `metadata.annotations.${ANNOTATION_LOCATION}`, + original_value: 'ignored', + value: + 'url:https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml', + }, + // match tree in location + { + entity_id: '5', + key: `metadata.annotations.${ANNOTATION_LOCATION}`, + original_value: 'ignored', + value: + 'url:https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml', + }, + // match blob in origin location + { + entity_id: '6', + key: `metadata.annotations.${ANNOTATION_ORIGIN_LOCATION}`, + original_value: 'ignored', + value: + 'url:https://github.com/backstage/demo/blob/master/folder/catalog-info.yaml', + }, + // match tree in origin location + { + entity_id: '7', + key: `metadata.annotations.${ANNOTATION_ORIGIN_LOCATION}`, + original_value: 'ignored', + value: + 'url:https://github.com/backstage/demo/tree/master/folder/catalog-info.yaml', + }, + // skip different but similar repo + { + entity_id: '8', + key: `metadata.annotations.${ANNOTATION_LOCATION}`, + original_value: 'ignored', + value: + 'url:https://github.com/backstage/demo2/tree/master/folder/catalog-info.yaml', + }, + ]); + + // Act + await subscriber!.onEvents([ + { + type: 'repository.updated', + url: 'https://github.com/backstage/demo', + }, + ]); + + // Verify + await expect( + knex('refresh_state') + .select('entity_id') + .where('next_update_at', '>', new Date('1981-01-01T00:00:00Z')) + .orderBy('entity_id', 'asc'), + ).resolves.toEqual([ + { entity_id: '1' }, + { entity_id: '2' }, + { entity_id: '4' }, + { entity_id: '5' }, + { entity_id: '6' }, + { entity_id: '7' }, + ]); + }); + }); +}); diff --git a/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.ts b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.ts new file mode 100644 index 0000000000..8f21bbf9de --- /dev/null +++ b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.ts @@ -0,0 +1,162 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, +} from '@backstage/catalog-model'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; +import { + CatalogScmEvent, + CatalogScmEventsService, +} from '@backstage/plugin-catalog-node/alpha'; +import { Knex } from 'knex'; +import { chunk } from 'lodash'; +import { + DbRefreshKeysRow, + DbRefreshStateRow, + DbSearchRow, +} from '../database/tables'; + +/** + * Deals in a generic fashion with SCM events, refreshing entities as needed. + * + * It's implemented in the form of an entity provider even though itt actually + * does not behave like one, mostly in order to consistently start treating + * events on connect time when the engine is known to be ready. + */ +export class GenericScmEventRefreshProvider implements EntityProvider { + readonly #knex: Knex; + readonly #scmEvents: CatalogScmEventsService; + + constructor(knex: Knex, scmEvents: CatalogScmEventsService) { + this.#knex = knex; + this.#scmEvents = scmEvents; + } + + getProviderName(): string { + return 'GenericScmEventRefreshProvider'; + } + + async connect(_connection: EntityProviderConnection): Promise { + this.#scmEvents.subscribe({ onEvents: this.#onScmEvents.bind(this) }); + } + + async #onScmEvents(events: CatalogScmEvent[]): Promise { + const exactLocationsToRefresh = new Set(); + const locationPrefixesToRefresh = new Set(); + + for (const event of events) { + if (event.type === 'location.updated') { + exactLocationsToRefresh.add(event.url); + } else if (event.type === 'repository.updated') { + if (!event.url.match(/[?#]/)) { + // TODO(freben): We can't yet support complex URL locations where e.g. + // the path can be anywhere in the URL including in the query or hash + // part. The code below currently assumes that we can use simple + // substring operations. + locationPrefixesToRefresh.add(event.url.replace(/\/*$/g, '/')); + } + } + } + + let count = 0; + + // Perform exact location updates + for (const urls of chunk( + expandUrlVariations(exactLocationsToRefresh), + 50, + )) { + const result = await this.#knex('refresh_state') + .update({ next_update_at: this.#knex.fn.now() }) + .whereIn('entity_id', keysQuery => { + return keysQuery + .table('refresh_keys') + .select('entity_id') + .whereIn('key', urls) + .union(searchQuery => { + return searchQuery + .table('search') + .select('entity_id') + .whereIn('key', [ + `metadata.annotations.${ANNOTATION_LOCATION}`, + `metadata.annotations.${ANNOTATION_ORIGIN_LOCATION}`, + ]) + .whereIn('value', urls); + }); + }); + + count += Number(result); + } + + // Perform prefix updates + for (const prefixes of chunk( + expandUrlVariations(locationPrefixesToRefresh), + 10, + )) { + const result = await this.#knex('refresh_state') + .update({ next_update_at: this.#knex.fn.now() }) + .whereIn('entity_id', keysQuery => { + return keysQuery + .table('refresh_keys') + .select('entity_id') + .where(inner => + prefixes.reduce( + (acc, prefix) => acc.orWhere('key', 'like', `${prefix}%`), + inner, + ), + ) + .union(searchQuery => { + return searchQuery + .table('search') + .select('entity_id') + .whereIn('key', [ + `metadata.annotations.${ANNOTATION_LOCATION}`, + `metadata.annotations.${ANNOTATION_ORIGIN_LOCATION}`, + ]) + .where(inner => + prefixes.reduce( + (acc, prefix) => acc.orWhere('value', 'like', `${prefix}%`), + inner, + ), + ); + }); + }); + + count += Number(result); + } + } +} + +/** + * Given a URL, returns all variations of that URL that may come into play for + * refreshes. + * + * This covers the oddity that the catalog sometimes uses `/tree/` instead of + * `/blob/` GitHub URLs, and in some circumstances there's a `url:` prefix and + * sometimes not. + */ +function expandUrlVariations(urls: Iterable): string[] { + const variations = new Set(); + for (const url of urls) { + variations.add(`url:${url}`); + variations.add(`url:${url.replace('/blob/', '/tree/')}`); + } + return Array.from(variations); +} diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index aa298d9198..db9ef8a867 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -112,8 +112,10 @@ import { entitiesResponseToObjects } from './response'; import { catalogEntityPermissionResourceRef, CatalogPermissionRuleInput, + CatalogScmEventsService, } from '@backstage/plugin-catalog-node/alpha'; import { filterAndSortProcessors, filterProviders } from './util'; +import { GenericScmEventRefreshProvider } from '../providers/GenericScmEventRefreshProvider'; export type CatalogEnvironment = { logger: LoggerService; @@ -127,6 +129,7 @@ export type CatalogEnvironment = { httpAuth: HttpAuthService; auditor: AuditorService; events: EventsService; + catalogScmEvents: CatalogScmEventsService; }; /** @@ -424,6 +427,7 @@ export class CatalogBuilder { auth, httpAuth, events, + catalogScmEvents, } = this.env; const enableRelationsCompatibility = Boolean( @@ -528,13 +532,19 @@ export class CatalogBuilder { }); } - const locationStore = new DefaultLocationStore(dbClient); + const locationStore = new DefaultLocationStore(dbClient, catalogScmEvents); const configLocationProvider = new ConfigLocationEntityProvider(config); + const scmEvents = new GenericScmEventRefreshProvider( + dbClient, + catalogScmEvents, + ); + const entityProviderEntries = lodash.uniqBy( [ ...this.entityProviders, { provider: locationStore }, { provider: configLocationProvider }, + { provider: scmEvents }, ], entry => entry.provider.getProviderName(), ); diff --git a/plugins/catalog-backend/src/service/CatalogPlugin.ts b/plugins/catalog-backend/src/service/CatalogPlugin.ts index a774d79457..26b09b9eea 100644 --- a/plugins/catalog-backend/src/service/CatalogPlugin.ts +++ b/plugins/catalog-backend/src/service/CatalogPlugin.ts @@ -39,6 +39,7 @@ import { CatalogPermissionExtensionPoint, catalogPermissionExtensionPoint, CatalogPermissionRuleInput, + catalogScmEventsServiceRef, } from '@backstage/plugin-catalog-node/alpha'; import { eventsServiceRef } from '@backstage/plugin-events-node'; import { Permission } from '@backstage/plugin-permission-common'; @@ -217,6 +218,7 @@ export const catalogPlugin = createBackendPlugin({ events: eventsServiceRef, catalog: catalogServiceRef, actionsRegistry: actionsRegistryServiceRef, + catalogScmEvents: catalogScmEventsServiceRef, }, async init({ logger, @@ -234,6 +236,7 @@ export const catalogPlugin = createBackendPlugin({ actionsRegistry, auditor, events, + catalogScmEvents, }) { const builder = await CatalogBuilder.create({ config, @@ -247,6 +250,7 @@ export const catalogPlugin = createBackendPlugin({ httpAuth, auditor, events, + catalogScmEvents, }); if (onProcessingError) { diff --git a/plugins/catalog-node/report-alpha.api.md b/plugins/catalog-node/report-alpha.api.md index c8ba44717b..2632184e46 100644 --- a/plugins/catalog-node/report-alpha.api.md +++ b/plugins/catalog-node/report-alpha.api.md @@ -73,6 +73,76 @@ export type CatalogProcessingExtensionPoint = CatalogProcessingExtensionPoint_2; // @alpha @deprecated (undocumented) export const catalogProcessingExtensionPoint: ExtensionPoint; +// @alpha +export type CatalogScmEvent = + | { + type: 'location.created'; + url: string; + context?: CatalogScmEventContext; + } + | { + type: 'location.updated'; + url: string; + context?: CatalogScmEventContext; + } + | { + type: 'location.deleted'; + url: string; + context?: CatalogScmEventContext; + } + | { + type: 'location.moved'; + fromUrl: string; + toUrl: string; + context?: CatalogScmEventContext; + } + | { + type: 'repository.created'; + url: string; + context?: CatalogScmEventContext; + } + | { + type: 'repository.updated'; + url: string; + context?: CatalogScmEventContext; + } + | { + type: 'repository.deleted'; + url: string; + context?: CatalogScmEventContext; + } + | { + type: 'repository.moved'; + fromUrl: string; + toUrl: string; + context?: CatalogScmEventContext; + }; + +// @alpha +export type CatalogScmEventContext = { + commitUrl?: string; +}; + +// @alpha +export interface CatalogScmEventsService { + publish(events: CatalogScmEvent[]): Promise; + subscribe(subscriber: CatalogScmEventsServiceSubscriber): { + unsubscribe: () => void; + }; +} + +// @alpha +export const catalogScmEventsServiceRef: ServiceRef< + CatalogScmEventsService, + 'root', + 'singleton' +>; + +// @alpha +export interface CatalogScmEventsServiceSubscriber { + onEvents: (events: CatalogScmEvent[]) => Promise; +} + // @alpha @deprecated (undocumented) export const catalogServiceRef: ServiceRef; diff --git a/plugins/catalog-node/src/alpha.ts b/plugins/catalog-node/src/alpha.ts index 8b42be5a95..8ed55f87ce 100644 --- a/plugins/catalog-node/src/alpha.ts +++ b/plugins/catalog-node/src/alpha.ts @@ -103,3 +103,5 @@ export { catalogModelExtensionPoint } from './extensions'; export type { CatalogPermissionRuleInput } from './extensions'; export type { CatalogPermissionExtensionPoint } from './extensions'; export { catalogPermissionExtensionPoint } from './extensions'; + +export * from './scmEvents'; diff --git a/plugins/catalog-node/src/scmEvents/DefaultCatalogScmEventsService.test.ts b/plugins/catalog-node/src/scmEvents/DefaultCatalogScmEventsService.test.ts new file mode 100644 index 0000000000..4adbb3b736 --- /dev/null +++ b/plugins/catalog-node/src/scmEvents/DefaultCatalogScmEventsService.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createDeferred } from '@backstage/types'; +import { DefaultCatalogScmEventsService } from './DefaultCatalogScmEventsService'; + +describe('DefaultCatalogScmEventsService', () => { + it('should publish and subscribe to events', async () => { + const service = new DefaultCatalogScmEventsService(); + + const subscriber1 = { + onEvents: jest.fn(), + }; + const subscriber2 = { + onEvents: jest.fn(), + }; + + service.subscribe(subscriber1); + service.subscribe(subscriber2); + + await service.publish([ + { + type: 'location.created', + url: 'https://github.com/backstage/backstage', + }, + ]); + + expect(subscriber1.onEvents).toHaveBeenCalledWith([ + { + type: 'location.created', + url: 'https://github.com/backstage/backstage', + }, + ]); + expect(subscriber2.onEvents).toHaveBeenCalledWith([ + { + type: 'location.created', + url: 'https://github.com/backstage/backstage', + }, + ]); + }); + + it('waits for all subscribers to acknowledge the events', async () => { + const service = new DefaultCatalogScmEventsService(); + + const work1 = createDeferred(); + const work2 = createDeferred(); + + const subscriber1 = { + onEvents: jest.fn().mockImplementation(async () => { + await work1; + }), + }; + const subscriber2 = { + onEvents: jest.fn().mockImplementation(async () => { + await work2; + }), + }; + + service.subscribe(subscriber1); + service.subscribe(subscriber2); + + let completed = false; + service + .publish([ + { + type: 'location.created', + url: 'https://github.com/backstage/backstage', + }, + ]) + .then(() => { + completed = true; + }); + + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(completed).toBe(false); + expect(subscriber1.onEvents).toHaveBeenCalled(); + expect(subscriber2.onEvents).toHaveBeenCalled(); + + work1.resolve(); + + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(completed).toBe(false); + + work2.resolve(); + + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(completed).toBe(true); + }); +}); diff --git a/plugins/catalog-node/src/scmEvents/DefaultCatalogScmEventsService.ts b/plugins/catalog-node/src/scmEvents/DefaultCatalogScmEventsService.ts new file mode 100644 index 0000000000..d0f975770a --- /dev/null +++ b/plugins/catalog-node/src/scmEvents/DefaultCatalogScmEventsService.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + CatalogScmEvent, + CatalogScmEventsService, + CatalogScmEventsServiceSubscriber, +} from './types'; + +/** + * The default implementation of the {@link CatalogScmEventsService}/{@link catalogScmEventsServiceRef}. + * + * @internal + * @remarks + * + * This implementation is in-memory, which requires the produceers and consumer + * (the catalog backend) to be deployed together. + */ +export class DefaultCatalogScmEventsService implements CatalogScmEventsService { + readonly #subscribers: Set; + + constructor() { + this.#subscribers = new Set(); + } + + subscribe(subscriber: CatalogScmEventsServiceSubscriber): { + unsubscribe: () => void; + } { + this.#subscribers.add(subscriber); + return { + unsubscribe: () => { + this.#subscribers.delete(subscriber); + }, + }; + } + + async publish(events: CatalogScmEvent[]): Promise { + await Promise.all( + Array.from(this.#subscribers).map(async subscriber => { + try { + await subscriber.onEvents(events); + } catch (error) { + // The subscribers are expected to handle errors themselves. + } + }), + ); + } +} diff --git a/plugins/catalog-node/src/scmEvents/catalogScmEventsServiceRef.ts b/plugins/catalog-node/src/scmEvents/catalogScmEventsServiceRef.ts new file mode 100644 index 0000000000..631d5b80c1 --- /dev/null +++ b/plugins/catalog-node/src/scmEvents/catalogScmEventsServiceRef.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + createServiceFactory, + createServiceRef, +} from '@backstage/backend-plugin-api'; +import { CatalogScmEventsService } from './types'; +import { DefaultCatalogScmEventsService } from './DefaultCatalogScmEventsService'; + +/** + * A service that allows publishing and subscribing to source control management + * system events. + * + * @alpha + * @remarks + * + * The default implementation of this service acts in-memory, which requires the + * produceers and consumer (the catalog backend) to be deployed together. + */ +export const catalogScmEventsServiceRef = + createServiceRef({ + id: 'catalog-scm-events', + scope: 'root', + defaultFactory: async service => + createServiceFactory({ + service, + deps: {}, + factory() { + return new DefaultCatalogScmEventsService(); + }, + }), + }); diff --git a/plugins/catalog-node/src/scmEvents/index.ts b/plugins/catalog-node/src/scmEvents/index.ts new file mode 100644 index 0000000000..66ad1c0b88 --- /dev/null +++ b/plugins/catalog-node/src/scmEvents/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { catalogScmEventsServiceRef } from './catalogScmEventsServiceRef'; +export type { + CatalogScmEvent, + CatalogScmEventContext, + CatalogScmEventsService, + CatalogScmEventsServiceSubscriber, +} from './types'; diff --git a/plugins/catalog-node/src/scmEvents/types.ts b/plugins/catalog-node/src/scmEvents/types.ts new file mode 100644 index 0000000000..91ad6f62de --- /dev/null +++ b/plugins/catalog-node/src/scmEvents/types.ts @@ -0,0 +1,175 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A subscriber of the {@link CatalogScmEventsService}. + * + * @alpha + */ +export interface CatalogScmEventsServiceSubscriber { + /** + * Receives a number of events. + */ + onEvents: (events: CatalogScmEvent[]) => Promise; +} + +/** + * A publish/subscribe service for source control management system events. This + * allows different producers of interesting events in a multi-SCM environment + * communicate those events to multiple interested parties. As an example, one + * entity provider might automatically register and unregister locations as an + * effect of these events. + * + * @alpha + */ +export interface CatalogScmEventsService { + /** + * Subscribes to events, and returns a function to unsubscribe. + */ + subscribe(subscriber: CatalogScmEventsServiceSubscriber): { + unsubscribe: () => void; + }; + + /** + * Publish an event to all subscribers. Returns once all subscribers have + * acknowledged that they have received and handled the event. + */ + publish(events: CatalogScmEvent[]): Promise; +} + +/** + * Voluntary contextual information related to a {@link CatalogScmEvent}. + * + * @alpha + */ +export type CatalogScmEventContext = { + /** + * URL to a commit related to this event being generated, if relevant. + */ + commitUrl?: string; +}; + +/** + * Represents a high level change event that happened in a source control + * management system. THese are usually produced as a distilled version of an + * incoming webhook event or similar. + * + * @alpha + */ +export type CatalogScmEvent = + | { + /** + * A new location was created. + * + * @remarks + * + * This typically means that an individual file was created in an existing + * repository, for example through a git push or merge. + */ + type: 'location.created'; + url: string; + context?: CatalogScmEventContext; + } + | { + /** + * An existing location was modified. + * + * @remarks + * + * This typically means that an individual file was modified in an existing + * repository, for example through a git push or merge. + */ + type: 'location.updated'; + url: string; + context?: CatalogScmEventContext; + } + | { + /** + * An existing location was deleted. + * + * @remarks + * + * This typically means that an individual file was created in an existing + * repository, for example through a git push or merge. + */ + type: 'location.deleted'; + url: string; + context?: CatalogScmEventContext; + } + | { + /** + * An existing location was moved from one place to another. + * + * @remarks + * + * This typically means that an individual file was moved or renamed, for + * example through a git push or merge. The URLs do not necessarily refer + * to the same repository before and after the move. + */ + type: 'location.moved'; + fromUrl: string; + toUrl: string; + context?: CatalogScmEventContext; + } + | { + /** + * A new repository was created. + */ + type: 'repository.created'; + url: string; + context?: CatalogScmEventContext; + } + | { + /** + * An existing repository was updated. + * + * @remarks + * + * This usually refers to some form of meta state change, such as it being + * made public or private, or its visibility being changed. + */ + type: 'repository.updated'; + url: string; + context?: CatalogScmEventContext; + } + | { + /** + * An existing repository was deleted. + */ + type: 'repository.deleted'; + url: string; + context?: CatalogScmEventContext; + } + | { + /** + * An existing repository was moved or in some other way had its effective + * base URL changed. + * + * @remarks + * + * This typically refres to a repository being renamed, or transferred to + * a different owner. It can also refer to a change of base branch, which + * effectively changes the base URL for many repository URL patterns. + * + * The source and target URLs do not necessarily end exactly on a + * repository, but MAY include additional path segments such as the branch + * name. + */ + type: 'repository.moved'; + fromUrl: string; + toUrl: string; + context?: CatalogScmEventContext; + }; diff --git a/yarn.lock b/yarn.lock index a1c84f2f09..fb6a6b6654 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5028,24 +5028,34 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github" dependencies: + "@backstage/backend-defaults": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-backend": "workspace:^" + "@backstage/plugin-events-backend-module-github": "workspace:^" + "@backstage/plugin-events-backend-module-google-pubsub": "workspace:^" "@backstage/plugin-events-node": "workspace:^" + "@backstage/types": "workspace:^" + "@octokit/auth-callback": "npm:^5.0.0" "@octokit/core": "npm:^5.2.0" "@octokit/graphql": "npm:^7.0.2" "@octokit/plugin-throttling": "npm:^8.1.3" "@octokit/rest": "npm:^19.0.3" + "@octokit/webhooks-types": "npm:^7.6.1" "@types/lodash": "npm:^4.14.151" git-url-parse: "npm:^15.0.0" lodash: "npm:^4.17.21" minimatch: "npm:^9.0.0" msw: "npm:^2.0.0" + octokit: "npm:^3.0.0" type-fest: "npm:^4.41.0" uuid: "npm:^11.0.0" languageName: unknown @@ -5797,7 +5807,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-events-backend-module-github@workspace:plugins/events-backend-module-github": +"@backstage/plugin-events-backend-module-github@workspace:^, @backstage/plugin-events-backend-module-github@workspace:plugins/events-backend-module-github": version: 0.0.0-use.local resolution: "@backstage/plugin-events-backend-module-github@workspace:plugins/events-backend-module-github" dependencies: @@ -12867,7 +12877,7 @@ __metadata: languageName: node linkType: hard -"@octokit/webhooks-types@npm:7.6.1": +"@octokit/webhooks-types@npm:7.6.1, @octokit/webhooks-types@npm:^7.6.1": version: 7.6.1 resolution: "@octokit/webhooks-types@npm:7.6.1" checksum: 10/0b11bd7e8d13b5a9cf14214421298a423d0180a5e1aaaea876ee4db6f97b5cca536f48d89af63105db75419d777a2402733eb0e110002d4dd59581ef36037bdc