From 3bce5789974ceb3fc80fecabe430d83d57bf6e6a Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Tue, 8 Apr 2025 22:31:06 +0200 Subject: [PATCH] feat(catalog): trigger entity updates based on Bitbucket Cloud's `repo:updated` events Signed-off-by: Patrick Jungermann --- .changeset/lovely-cats-take.md | 11 + docs/integrations/bitbucketCloud/discovery.md | 17 ++ .../BitbucketCloudEntityProvider.test.ts | 255 ++++++++++++++++++ .../providers/BitbucketCloudEntityProvider.ts | 81 ++++-- 4 files changed, 347 insertions(+), 17 deletions(-) create mode 100644 .changeset/lovely-cats-take.md diff --git a/.changeset/lovely-cats-take.md b/.changeset/lovely-cats-take.md new file mode 100644 index 0000000000..ab1ee1c249 --- /dev/null +++ b/.changeset/lovely-cats-take.md @@ -0,0 +1,11 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch +--- + +Support Bitbucket Cloud's `repo:updated` events at `BitbucketCloudEntityProvider`. + +To make use of the new event type, you have to configure your webhook or add a new ones +that delivers this event type to Backstage similar to `repo:push` before. + +Only `repo:updated` events that modify a repository's URL (e.g., due to a name change) +will cause changes (removing the "old", adding the "new" repository). diff --git a/docs/integrations/bitbucketCloud/discovery.md b/docs/integrations/bitbucketCloud/discovery.md index ff0e60f498..9560c5e06b 100644 --- a/docs/integrations/bitbucketCloud/discovery.md +++ b/docs/integrations/bitbucketCloud/discovery.md @@ -12,6 +12,23 @@ The provider will search your Bitbucket Cloud account and register catalog files as Location entity and via following processing steps add all contained catalog entities. This can be useful as an alternative to static locations or manually adding things to the catalog. +## Event-based Discovery + +Supported events for event-based updates: + +- [`repo:push`](https://support.atlassian.com/bitbucket-cloud/docs/event-payloads/#Push) + received as `bitbucket.repo:push` +- [`repo:updated`](https://support.atlassian.com/bitbucket-cloud/docs/event-payloads/#Updated) + received as `bitbucket.repo:updated` + - Changes are triggered if the repository slug/name changed, causing a new URL for it. + +To receive events, you need to add webhook subscriptions at Bitbucket Cloud. +Enable the triggers (event types) you want to receive. +For the entity provider, only the event types above are supported ("Repository Push" and/or "Repository Updated") +and additional event types will be ignored (e.g., for other use cases/integrations you might have). + +For more information on how to set up the event-based discovery, see the installation instructions below. + ## Installation You will have to add the entity provider in the catalog initialization code of your diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts index 5b2f9992c9..6ea92dac96 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts @@ -123,6 +123,55 @@ describe('BitbucketCloudEntityProvider', () => { eventPayload: repoPushEvent, metadata: { 'x-event-key': 'repo:push' }, }; + const repoUpdatedEvent: Events.RepoUpdatedEvent = { + actor: { + type: 'user', + }, + repository: { + type: 'repository', + full_name: 'test-ws/test-repo-new', + links: { + html: { + href: 'https://bitbucket.org/test-ws/test-repo-new', + }, + }, + workspace: { + type: 'workspace', + slug: 'test-ws', + }, + project: { + type: 'project', + key: 'test-project', + }, + }, + changes: { + name: { + new: 'test-repo-new', + old: 'test-repo-old', + }, + links: { + new: { + html: { + href: 'https://bitbucket.org/test-ws/test-repo-new', + }, + }, + old: { + html: { + href: 'https://bitbucket.org/test-ws/test-repo-old', + }, + }, + }, + full_name: { + new: 'test-ws/test-repo-new', + old: 'test-ws/test-repo-old', + }, + }, + }; + const repoUpdatedEventParams = { + topic: 'bitbucketCloud.repo:updated', + eventPayload: repoUpdatedEvent, + metadata: { 'x-event-key': 'repo:updated' }, + }; const createLocationEntity = ( repoUrl: string, @@ -705,4 +754,210 @@ describe('BitbucketCloudEntityProvider', () => { expect(catalogApi.refreshEntity).toHaveBeenCalledTimes(0); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); }); + + it('update onRepoUpdated', async () => { + const oldModule = createLocationEntity( + 'https://bitbucket.org/test-ws/test-repo-old', + 'main', + 'module/catalog-custom.yaml', + ); + const newModule = createLocationEntity( + 'https://bitbucket.org/test-ws/test-repo-new', + 'main', + 'module/catalog-custom.yaml', + ); + + const auth = mockServices.auth.mock({ + getPluginRequestToken: async () => ({ token: 'fake-token' }), + }); + const events = DefaultEventsService.create({ logger }); + const catalogApi = catalogServiceMock.mock({ + getEntities: async ( + request: { filter: Record }, + options: { token: string }, + ): Promise<{ items: Entity[] }> => { + if ( + options.token !== 'fake-token' || + request.filter.kind !== 'Location' || + request.filter['metadata.annotations.bitbucket.org/repo-url'] !== + 'https://bitbucket.org/test-ws/test-repo-old' + ) { + return { items: [] }; + } + + return { + items: [oldModule], + }; + }, + }); + const provider = BitbucketCloudEntityProvider.fromConfig(defaultConfig, { + auth, + catalogApi, + events, + logger, + schedule, + })[0]; + + server.use( + rest.get( + `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, + (req, res, ctx) => { + const query = req.url.searchParams.get('search_query'); + if (!query || !query.includes('repo:test-repo-new')) { + return res(ctx.json({ values: [] })); + } + + const response = { + values: [ + { + path_matches: [ + { + match: true, + text: 'catalog-custom.yaml', + }, + ], + file: { + type: 'commit_file', + path: 'module/catalog-custom.yaml', + commit: { + repository: { + slug: 'test-repo-new', + project: { + key: 'test-project', + }, + mainbranch: { + name: 'main', + }, + links: { + html: { + href: 'https://bitbucket.org/test-ws/test-repo-new', + }, + }, + }, + }, + }, + }, + ], + }; + return res(ctx.json(response)); + }, + ), + ); + + await provider.connect(entityProviderConnection); + await events.publish(repoUpdatedEventParams); + + const addedEntities = [ + { + entity: newModule, + locationKey: 'bitbucketCloud-provider:myProvider', + }, + ]; + const removedEntities = [ + { + entity: oldModule, + locationKey: 'bitbucketCloud-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: addedEntities, + removed: removedEntities, + }); + }); + + it('no onRepoUpdated update on non-matching workspace slug', async () => { + const auth = mockServices.auth.mock(); + const catalogApi = catalogServiceMock(); + jest.spyOn(catalogApi, 'refreshEntity'); + const events = DefaultEventsService.create({ logger }); + const provider = BitbucketCloudEntityProvider.fromConfig(defaultConfig, { + auth, + catalogApi, + events, + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + await events.publish({ + ...repoUpdatedEventParams, + eventPayload: { + ...repoUpdatedEventParams.eventPayload, + repository: { + ...repoUpdatedEventParams.eventPayload.repository, + workspace: { + ...repoUpdatedEventParams.eventPayload.repository.workspace, + slug: 'not-matching', + }, + }, + }, + }); + + expect(catalogApi.refreshEntity).toHaveBeenCalledTimes(0); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('no onRepoUpdated update on non-matching repo slug', async () => { + const auth = mockServices.auth.mock(); + const catalogApi = catalogServiceMock(); + jest.spyOn(catalogApi, 'refreshEntity'); + const events = DefaultEventsService.create({ logger }); + const provider = BitbucketCloudEntityProvider.fromConfig(defaultConfig, { + auth, + catalogApi, + events, + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + await events.publish({ + ...repoUpdatedEventParams, + eventPayload: { + ...repoUpdatedEventParams.eventPayload, + repository: { + ...repoUpdatedEventParams.eventPayload.repository, + full_name: `${repoUpdatedEventParams.eventPayload.repository.workspace.slug}/not-matching`, + }, + }, + }); + + expect(catalogApi.refreshEntity).toHaveBeenCalledTimes(0); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('no onRepoUpdated update on non-relevant repo update', async () => { + const auth = mockServices.auth.mock(); + const catalogApi = catalogServiceMock(); + jest.spyOn(catalogApi, 'refreshEntity'); + const events = DefaultEventsService.create({ logger }); + const provider = BitbucketCloudEntityProvider.fromConfig(defaultConfig, { + auth, + catalogApi, + events, + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + await events.publish({ + ...repoUpdatedEventParams, + eventPayload: { + ...repoUpdatedEventParams.eventPayload, + changes: { + description: { + new: 'New description', + old: 'Old description', + }, + }, + }, + }); + + expect(catalogApi.refreshEntity).toHaveBeenCalledTimes(0); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); }); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts index 5beb98ac08..88d0cb80ef 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts @@ -48,6 +48,7 @@ import * as uuid from 'uuid'; const DEFAULT_BRANCH = 'master'; const TOPIC_REPO_PUSH = 'bitbucketCloud.repo:push'; +const TOPIC_REPO_UPDATED = 'bitbucketCloud.repo:updated'; /** @public */ export const ANNOTATION_BITBUCKET_CLOUD_REPO_URL = 'bitbucket.org/repo-url'; @@ -186,13 +187,17 @@ export class BitbucketCloudEntityProvider implements EntityProvider { await this.events.subscribe({ id: this.getProviderName(), - topics: [TOPIC_REPO_PUSH], + topics: [TOPIC_REPO_PUSH, TOPIC_REPO_UPDATED], onEvent: async params => { - if (params.topic !== TOPIC_REPO_PUSH) { - return; + if (params.topic === TOPIC_REPO_PUSH) { + await this.onRepoPush(params.eventPayload as Events.RepoPushEvent); } - await this.onRepoPush(params.eventPayload as Events.RepoPushEvent); + if (params.topic === TOPIC_REPO_UPDATED) { + await this.onRepoUpdated( + params.eventPayload as Events.RepoUpdatedEvent, + ); + } }, }); } @@ -217,12 +222,12 @@ export class BitbucketCloudEntityProvider implements EntityProvider { ); } - private enhanceEvent(event: Events.RepoPushEvent): void { + private enhanceEvent(event: Events.RepoEvent): void { // add missing slug event.repository.slug = event.repository.full_name!.split('/', 2)[1]; } - async onRepoPush(event: Events.RepoPushEvent): Promise { + private shouldProcessEvent(event: Events.RepoEvent): boolean { if (!this.connection) { throw new Error('Not initialized'); } @@ -230,10 +235,18 @@ export class BitbucketCloudEntityProvider implements EntityProvider { this.enhanceEvent(event); if (event.repository.workspace.slug !== this.config.workspace) { - return; + return false; } if (!this.matchesFilters(event.repository)) { + return false; + } + + return true; + } + + async onRepoPush(event: Events.RepoPushEvent): Promise { + if (!this.shouldProcessEvent(event)) { return; } @@ -248,12 +261,43 @@ export class BitbucketCloudEntityProvider implements EntityProvider { // Hence, we will just trigger a refresh for catalog file(s) within the repository // if we get notified about changes there. - const targets = await this.findCatalogFiles(repoSlug); - + const expected = await this.findCatalogFiles(repoSlug); const existing = await this.findExistingLocations(repoUrl); + await this.updateForChanges(expected, existing); + } + + private async onRepoUpdated(event: Events.RepoUpdatedEvent): Promise { + if (!this.shouldProcessEvent(event)) { + return; + } + + // The event is triggered on every change to the repository. + // We are only interested in changes that affect the repository URL. + // This is the case when the repository name (slug), or the full name changes. + if (!event.changes.links?.old.html?.href) { + return; + } + + const repoSlug = event.repository.slug!; + const repoUrl = event.repository.links!.html!.href!; + const oldRepoUrl = event.changes.links.old.html.href; + this.logger.info( + `handle repo:updated event for ${repoUrl}, previous: ${oldRepoUrl}`, + ); + + const expected = await this.findCatalogFiles(repoSlug); + const existing = await this.findExistingLocations(oldRepoUrl); + + await this.updateForChanges(expected, existing); + } + + private async updateForChanges( + expected: IngestionTarget[], + existing: LocationEntity[], + ): Promise { const added: DeferredEntity[] = this.toDeferredEntities( - targets.filter( + expected.filter( // All Locations are managed by this provider and only have `target`, never `targets`. // All URLs (fileUrl, target) are created using `BitbucketCloudEntityProvider.toUrl`. // Hence, we can keep the comparison simple and don't need to handle different @@ -265,7 +309,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { const stillExisting: LocationEntity[] = []; const removed: DeferredEntity[] = []; existing.forEach(item => { - if (targets.find(value => value.fileUrl === item.spec.target)) { + if (expected.find(value => value.fileUrl === item.spec.target)) { stillExisting.push(item); } else { removed.push({ @@ -275,14 +319,17 @@ export class BitbucketCloudEntityProvider implements EntityProvider { } }); - const promises: Promise[] = [ - this.connection.refresh({ - keys: stillExisting.map(entity => `url:${entity.spec.target}`), - }), - ]; + const promises: Promise[] = + stillExisting.length === 0 + ? [] + : [ + this.connection!.refresh({ + keys: stillExisting.map(entity => `url:${entity.spec.target}`), + }), + ]; if (added.length > 0 || removed.length > 0) { - const connection = this.connection; + const connection = this.connection!; promises.push( connection.applyMutation({ type: 'delta',