Merge pull request #29518 from Bonial-International-GmbH/pjungermann/bitbucket/repo-updated-event

catalog/bitbucket-cloud: support repo:updated events
2025-04-22 13:16:36 +02:00
parent 506cd51a77 3bce578997
commit 2b7937d44b
10 changed files with 2092 additions and 812 deletions
@@ -0,0 +1,17 @@
+---
+'@backstage/plugin-bitbucket-cloud-common': minor
+---
+
+Update Bitbucket Cloud schema and models.
+
+The latest schema was fetched from Bitbucket Cloud and stored locally.
+Based on the updated schema, the models got regenerated.
+
+**BREAKING:**
+
+Due to the schema changes, the model update includes one breaking change:
+
+- `Account.username` was removed.
+
+Additionally, there were a couple of compatible changes including the addition of
+`BaseCommit.committer` and others.
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-bitbucket-cloud-common': patch
+---
+
+Add support for `repo:updated` events as `Events.RepoUpdatedEvent`.
@@ -0,0 +1,11 @@
+---
+'@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch
+---
+
+Support Bitbucket Cloud's `repo:updated` events at `BitbucketCloudEntityProvider`.
+
+To make use of the new event type, you have to configure your webhook or add a new ones
+that delivers this event type to Backstage similar to `repo:push` before.
+
+Only `repo:updated` events that modify a repository's URL (e.g., due to a name change)
+will cause changes (removing the "old", adding the "new" repository).
@@ -12,6 +12,23 @@ The provider will search your Bitbucket Cloud account and register catalog files
 as Location entity and via following processing steps add all contained catalog entities.
 This can be useful as an alternative to static locations or manually adding things to the catalog.

+## Event-based Discovery
+
+Supported events for event-based updates:
+
+- [`repo:push`](https://support.atlassian.com/bitbucket-cloud/docs/event-payloads/#Push)
+  received as `bitbucket.repo:push`
+- [`repo:updated`](https://support.atlassian.com/bitbucket-cloud/docs/event-payloads/#Updated)
+  received as `bitbucket.repo:updated`
+  - Changes are triggered if the repository slug/name changed, causing a new URL for it.
+
+To receive events, you need to add webhook subscriptions at Bitbucket Cloud.
+Enable the triggers (event types) you want to receive.
+For the entity provider, only the event types above are supported ("Repository Push" and/or "Repository Updated")
+and additional event types will be ignored (e.g., for other use cases/integrations you might have).
+
+For more information on how to set up the event-based discovery, see the installation instructions below.
+
 ## Installation

 You will have to add the entity provider in the catalog initialization code of your
@@ -70,6 +70,30 @@ export namespace Events {
    html: Models.Link;
  }
  // (undocumented)
+  export interface RepoChange<T> {
+    // (undocumented)
+    new: T;
+    // (undocumented)
+    old: T;
+  }
+  // (undocumented)
+  export interface RepoChanges {
+    // (undocumented)
+    description?: RepoChange<string>;
+    // (undocumented)
+    full_name?: RepoChange<string>;
+    // (undocumented)
+    language?: RepoChange<string>;
+    // (undocumented)
+    links?: RepoChange<
+      Pick<Models.RepositoryLinks, 'avatar' | 'html' | 'self'>
+    >;
+    // (undocumented)
+    name?: RepoChange<string>;
+    // (undocumented)
+    website?: RepoChange<string>;
+  }
+  // (undocumented)
  export interface RepoEvent {
    // (undocumented)
    actor: Models.Account;
@@ -88,6 +112,11 @@ export namespace Events {
    // (undocumented)
    push: RepoPush;
  }
+  // (undocumented)
+  export interface RepoUpdatedEvent extends RepoEvent {
+    // (undocumented)
+    changes: RepoChanges;
+  }
 }

 // @public (undocumented)
@@ -106,8 +135,6 @@ export namespace Models {
    // (undocumented)
    links?: AccountLinks;
    // (undocumented)
-    username?: string;
-    // (undocumented)
    uuid?: string;
  }
  export interface AccountLinks {
@@ -125,6 +152,8 @@ export namespace Models {
    // (undocumented)
    author?: Author;
    // (undocumented)
+    committer?: Committer;
+    // (undocumented)
    date?: string;
    // (undocumented)
    hash?: string;
@@ -163,6 +192,9 @@ export namespace Models {
    readonly MergeCommit: 'merge_commit';
    readonly Squash: 'squash';
    readonly FastForward: 'fast_forward';
+    readonly SquashFastForward: 'squash_fast_forward';
+    readonly RebaseFastForward: 'rebase_fast_forward';
+    readonly RebaseMerge: 'rebase_merge';
  };
  export type BranchMergeStrategiesEnum =
    (typeof BranchMergeStrategiesEnum)[keyof typeof BranchMergeStrategiesEnum];
@@ -195,6 +227,11 @@ export namespace Models {
  // (undocumented)
  export type CommitFileAttributesEnum =
    (typeof CommitFileAttributesEnum)[keyof typeof CommitFileAttributesEnum];
+  export interface Committer extends ModelObject {
+    raw?: string;
+    // (undocumented)
+    user?: Account;
+  }
  export interface Link {
    // (undocumented)
    href?: string;
@@ -416,6 +453,8 @@ export namespace Models {
  export interface Workspace extends ModelObject {
    // (undocumented)
    created_on?: string;
+    forking_mode?: WorkspaceForkingModeEnum;
+    is_privacy_enforced?: boolean;
    is_private?: boolean;
    // (undocumented)
    links?: WorkspaceLinks;
@@ -425,6 +464,12 @@ export namespace Models {
    updated_on?: string;
    uuid?: string;
  }
+  const WorkspaceForkingModeEnum: {
+    readonly AllowForks: 'allow_forks';
+    readonly InternalOnly: 'internal_only';
+  };
+  export type WorkspaceForkingModeEnum =
+    (typeof WorkspaceForkingModeEnum)[keyof typeof WorkspaceForkingModeEnum];
  // (undocumented)
  export interface WorkspaceLinks {
    // (undocumented)
@@ -30,6 +30,29 @@ export namespace Events {
    push: RepoPush;
  }

+  /** @public */
+  export interface RepoUpdatedEvent extends RepoEvent {
+    changes: RepoChanges;
+  }
+
+  /** @public */
+  export interface RepoChanges {
+    description?: RepoChange<string>;
+    full_name?: RepoChange<string>;
+    language?: RepoChange<string>;
+    links?: RepoChange<
+      Pick<Models.RepositoryLinks, 'avatar' | 'html' | 'self'>
+    >;
+    name?: RepoChange<string>;
+    website?: RepoChange<string>;
+  }
+
+  /** @public */
+  export interface RepoChange<T> {
+    new: T;
+    old: T;
+  }
+
  /** @public */
  export interface RepoPush {
    changes: Change[];
@@ -36,7 +36,6 @@ export namespace Models {
    created_on?: string;
    display_name?: string;
    links?: AccountLinks;
-    username?: string;
    uuid?: string;
  }

@@ -67,6 +66,7 @@ export namespace Models {
   */
  export interface BaseCommit extends ModelObject {
    author?: Author;
+    committer?: Committer;
    date?: string;
    hash?: string;
    message?: string;
@@ -139,6 +139,9 @@ export namespace Models {
    MergeCommit: 'merge_commit',
    Squash: 'squash',
    FastForward: 'fast_forward',
+    SquashFastForward: 'squash_fast_forward',
+    RebaseFastForward: 'rebase_fast_forward',
+    RebaseMerge: 'rebase_merge',
  } as const;

  /**
@@ -193,6 +196,18 @@ export namespace Models {
  export type CommitFileAttributesEnum =
    (typeof CommitFileAttributesEnum)[keyof typeof CommitFileAttributesEnum];

+  /**
+   * The committer of a change in a repository
+   * @public
+   */
+  export interface Committer extends ModelObject {
+    /**
+     * The raw committer value from the repository. This may be the only value available if the committer does not match a user in Bitbucket.
+     */
+    raw?: string;
+    user?: Account;
+  }
+
  /**
   * A link to a resource related to this object.
   * @public
@@ -242,6 +257,28 @@ export namespace Models {
    values?: Array<TResultItem> | Set<TResultItem>;
  }

+  /**
+   * A paginated list of branches.
+   * @public
+   */
+  export interface PaginatedBranches extends Paginated<Branch> {
+    /**
+     * The values of the current page.
+     */
+    values?: Set<Branch>;
+  }
+
+  /**
+   * A paginated list of projects
+   * @public
+   */
+  export interface PaginatedProjects extends Paginated<Project> {
+    /**
+     * The values of the current page.
+     */
+    values?: Set<Project>;
+  }
+
  /**
   * A paginated list of repositories.
   * @public
@@ -253,17 +290,6 @@ export namespace Models {
    values?: Set<Repository>;
  }

-  /**
-   * A paginated list of projects.
-   * @public
-   */
-  export interface PaginatedProjects extends Paginated<Project> {
-    /**
-     * The values of the current page.
-     */
-    values?: Set<Project>;
-  }
-
  /**
   * A paginated list of workspaces.
   * @public
@@ -275,17 +301,6 @@ export namespace Models {
    values?: Set<Workspace>;
  }

-  /**
-   * A paginated list of branches.
-   * @public
-   */
-  export interface PaginatedBranches extends Paginated<Branch> {
-    /**
-     * The values of the current page.
-     */
-    values?: Set<Branch>;
-  }
-
  /**
   * Object describing a user's role on resources like commits or pull requests.
   * @public
@@ -571,6 +586,17 @@ export namespace Models {
   */
  export interface Workspace extends ModelObject {
    created_on?: string;
+    /**
+     * Controls the rules for forking repositories within this workspace.
+     *
+     * * **allow_forks**: unrestricted forking
+     * * **internal_only**: prevents forking of private repositories outside the workspace or to public repositories
+     */
+    forking_mode?: WorkspaceForkingModeEnum;
+    /**
+     * Indicates whether the workspace enforces private content, or whether it allows public content.
+     */
+    is_privacy_enforced?: boolean;
    /**
     * Indicates whether the workspace is publicly accessible, or whether it is
     * private to the members and consequently only visible to members.
@@ -592,6 +618,28 @@ export namespace Models {
    uuid?: string;
  }

+  /**
+   * Controls the rules for forking repositories within this workspace.
+   *
+   * * **allow_forks**: unrestricted forking
+   * * **internal_only**: prevents forking of private repositories outside the workspace or to public repositories
+   * @public
+   */
+  export const WorkspaceForkingModeEnum = {
+    AllowForks: 'allow_forks',
+    InternalOnly: 'internal_only',
+  } as const;
+
+  /**
+   * Controls the rules for forking repositories within this workspace.
+   *
+   * * **allow_forks**: unrestricted forking
+   * * **internal_only**: prevents forking of private repositories outside the workspace or to public repositories
+   * @public
+   */
+  export type WorkspaceForkingModeEnum =
+    (typeof WorkspaceForkingModeEnum)[keyof typeof WorkspaceForkingModeEnum];
+
  /**
   * @public
   */
@@ -123,6 +123,55 @@ describe('BitbucketCloudEntityProvider', () => {
    eventPayload: repoPushEvent,
    metadata: { 'x-event-key': 'repo:push' },
  };
+  const repoUpdatedEvent: Events.RepoUpdatedEvent = {
+    actor: {
+      type: 'user',
+    },
+    repository: {
+      type: 'repository',
+      full_name: 'test-ws/test-repo-new',
+      links: {
+        html: {
+          href: 'https://bitbucket.org/test-ws/test-repo-new',
+        },
+      },
+      workspace: {
+        type: 'workspace',
+        slug: 'test-ws',
+      },
+      project: {
+        type: 'project',
+        key: 'test-project',
+      },
+    },
+    changes: {
+      name: {
+        new: 'test-repo-new',
+        old: 'test-repo-old',
+      },
+      links: {
+        new: {
+          html: {
+            href: 'https://bitbucket.org/test-ws/test-repo-new',
+          },
+        },
+        old: {
+          html: {
+            href: 'https://bitbucket.org/test-ws/test-repo-old',
+          },
+        },
+      },
+      full_name: {
+        new: 'test-ws/test-repo-new',
+        old: 'test-ws/test-repo-old',
+      },
+    },
+  };
+  const repoUpdatedEventParams = {
+    topic: 'bitbucketCloud.repo:updated',
+    eventPayload: repoUpdatedEvent,
+    metadata: { 'x-event-key': 'repo:updated' },
+  };

  const createLocationEntity = (
    repoUrl: string,
@@ -705,4 +754,210 @@ describe('BitbucketCloudEntityProvider', () => {
    expect(catalogApi.refreshEntity).toHaveBeenCalledTimes(0);
    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
  });
+
+  it('update onRepoUpdated', async () => {
+    const oldModule = createLocationEntity(
+      'https://bitbucket.org/test-ws/test-repo-old',
+      'main',
+      'module/catalog-custom.yaml',
+    );
+    const newModule = createLocationEntity(
+      'https://bitbucket.org/test-ws/test-repo-new',
+      'main',
+      'module/catalog-custom.yaml',
+    );
+
+    const auth = mockServices.auth.mock({
+      getPluginRequestToken: async () => ({ token: 'fake-token' }),
+    });
+    const events = DefaultEventsService.create({ logger });
+    const catalogApi = catalogServiceMock.mock({
+      getEntities: async (
+        request: { filter: Record<string, string> },
+        options: { token: string },
+      ): Promise<{ items: Entity[] }> => {
+        if (
+          options.token !== 'fake-token' ||
+          request.filter.kind !== 'Location' ||
+          request.filter['metadata.annotations.bitbucket.org/repo-url'] !==
+            'https://bitbucket.org/test-ws/test-repo-old'
+        ) {
+          return { items: [] };
+        }
+
+        return {
+          items: [oldModule],
+        };
+      },
+    });
+    const provider = BitbucketCloudEntityProvider.fromConfig(defaultConfig, {
+      auth,
+      catalogApi,
+      events,
+      logger,
+      schedule,
+    })[0];
+
+    server.use(
+      rest.get(
+        `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`,
+        (req, res, ctx) => {
+          const query = req.url.searchParams.get('search_query');
+          if (!query || !query.includes('repo:test-repo-new')) {
+            return res(ctx.json({ values: [] }));
+          }
+
+          const response = {
+            values: [
+              {
+                path_matches: [
+                  {
+                    match: true,
+                    text: 'catalog-custom.yaml',
+                  },
+                ],
+                file: {
+                  type: 'commit_file',
+                  path: 'module/catalog-custom.yaml',
+                  commit: {
+                    repository: {
+                      slug: 'test-repo-new',
+                      project: {
+                        key: 'test-project',
+                      },
+                      mainbranch: {
+                        name: 'main',
+                      },
+                      links: {
+                        html: {
+                          href: 'https://bitbucket.org/test-ws/test-repo-new',
+                        },
+                      },
+                    },
+                  },
+                },
+              },
+            ],
+          };
+          return res(ctx.json(response));
+        },
+      ),
+    );
+
+    await provider.connect(entityProviderConnection);
+    await events.publish(repoUpdatedEventParams);
+
+    const addedEntities = [
+      {
+        entity: newModule,
+        locationKey: 'bitbucketCloud-provider:myProvider',
+      },
+    ];
+    const removedEntities = [
+      {
+        entity: oldModule,
+        locationKey: 'bitbucketCloud-provider:myProvider',
+      },
+    ];
+
+    expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'delta',
+      added: addedEntities,
+      removed: removedEntities,
+    });
+  });
+
+  it('no onRepoUpdated update on non-matching workspace slug', async () => {
+    const auth = mockServices.auth.mock();
+    const catalogApi = catalogServiceMock();
+    jest.spyOn(catalogApi, 'refreshEntity');
+    const events = DefaultEventsService.create({ logger });
+    const provider = BitbucketCloudEntityProvider.fromConfig(defaultConfig, {
+      auth,
+      catalogApi,
+      events,
+      logger,
+      schedule,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+    await events.publish({
+      ...repoUpdatedEventParams,
+      eventPayload: {
+        ...repoUpdatedEventParams.eventPayload,
+        repository: {
+          ...repoUpdatedEventParams.eventPayload.repository,
+          workspace: {
+            ...repoUpdatedEventParams.eventPayload.repository.workspace,
+            slug: 'not-matching',
+          },
+        },
+      },
+    });
+
+    expect(catalogApi.refreshEntity).toHaveBeenCalledTimes(0);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+
+  it('no onRepoUpdated update on non-matching repo slug', async () => {
+    const auth = mockServices.auth.mock();
+    const catalogApi = catalogServiceMock();
+    jest.spyOn(catalogApi, 'refreshEntity');
+    const events = DefaultEventsService.create({ logger });
+    const provider = BitbucketCloudEntityProvider.fromConfig(defaultConfig, {
+      auth,
+      catalogApi,
+      events,
+      logger,
+      schedule,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+    await events.publish({
+      ...repoUpdatedEventParams,
+      eventPayload: {
+        ...repoUpdatedEventParams.eventPayload,
+        repository: {
+          ...repoUpdatedEventParams.eventPayload.repository,
+          full_name: `${repoUpdatedEventParams.eventPayload.repository.workspace.slug}/not-matching`,
+        },
+      },
+    });
+
+    expect(catalogApi.refreshEntity).toHaveBeenCalledTimes(0);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+
+  it('no onRepoUpdated update on non-relevant repo update', async () => {
+    const auth = mockServices.auth.mock();
+    const catalogApi = catalogServiceMock();
+    jest.spyOn(catalogApi, 'refreshEntity');
+    const events = DefaultEventsService.create({ logger });
+    const provider = BitbucketCloudEntityProvider.fromConfig(defaultConfig, {
+      auth,
+      catalogApi,
+      events,
+      logger,
+      schedule,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+    await events.publish({
+      ...repoUpdatedEventParams,
+      eventPayload: {
+        ...repoUpdatedEventParams.eventPayload,
+        changes: {
+          description: {
+            new: 'New description',
+            old: 'Old description',
+          },
+        },
+      },
+    });
+
+    expect(catalogApi.refreshEntity).toHaveBeenCalledTimes(0);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
 });
@@ -48,6 +48,7 @@ import * as uuid from 'uuid';

 const DEFAULT_BRANCH = 'master';
 const TOPIC_REPO_PUSH = 'bitbucketCloud.repo:push';
+const TOPIC_REPO_UPDATED = 'bitbucketCloud.repo:updated';

 /** @public */
 export const ANNOTATION_BITBUCKET_CLOUD_REPO_URL = 'bitbucket.org/repo-url';
@@ -186,13 +187,17 @@ export class BitbucketCloudEntityProvider implements EntityProvider {

    await this.events.subscribe({
      id: this.getProviderName(),
-      topics: [TOPIC_REPO_PUSH],
+      topics: [TOPIC_REPO_PUSH, TOPIC_REPO_UPDATED],
      onEvent: async params => {
-        if (params.topic !== TOPIC_REPO_PUSH) {
-          return;
+        if (params.topic === TOPIC_REPO_PUSH) {
+          await this.onRepoPush(params.eventPayload as Events.RepoPushEvent);
        }

-        await this.onRepoPush(params.eventPayload as Events.RepoPushEvent);
+        if (params.topic === TOPIC_REPO_UPDATED) {
+          await this.onRepoUpdated(
+            params.eventPayload as Events.RepoUpdatedEvent,
+          );
+        }
      },
    });
  }
@@ -217,12 +222,12 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
    );
  }

-  private enhanceEvent(event: Events.RepoPushEvent): void {
+  private enhanceEvent(event: Events.RepoEvent): void {
    // add missing slug
    event.repository.slug = event.repository.full_name!.split('/', 2)[1];
  }

-  async onRepoPush(event: Events.RepoPushEvent): Promise<void> {
+  private shouldProcessEvent(event: Events.RepoEvent): boolean {
    if (!this.connection) {
      throw new Error('Not initialized');
    }
@@ -230,10 +235,18 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
    this.enhanceEvent(event);

    if (event.repository.workspace.slug !== this.config.workspace) {
-      return;
+      return false;
    }

    if (!this.matchesFilters(event.repository)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  async onRepoPush(event: Events.RepoPushEvent): Promise<void> {
+    if (!this.shouldProcessEvent(event)) {
      return;
    }

@@ -248,12 +261,43 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
    // Hence, we will just trigger a refresh for catalog file(s) within the repository
    // if we get notified about changes there.

-    const targets = await this.findCatalogFiles(repoSlug);
-
+    const expected = await this.findCatalogFiles(repoSlug);
    const existing = await this.findExistingLocations(repoUrl);

+    await this.updateForChanges(expected, existing);
+  }
+
+  private async onRepoUpdated(event: Events.RepoUpdatedEvent): Promise<void> {
+    if (!this.shouldProcessEvent(event)) {
+      return;
+    }
+
+    // The event is triggered on every change to the repository.
+    // We are only interested in changes that affect the repository URL.
+    // This is the case when the repository name (slug), or the full name changes.
+    if (!event.changes.links?.old.html?.href) {
+      return;
+    }
+
+    const repoSlug = event.repository.slug!;
+    const repoUrl = event.repository.links!.html!.href!;
+    const oldRepoUrl = event.changes.links.old.html.href;
+    this.logger.info(
+      `handle repo:updated event for ${repoUrl}, previous: ${oldRepoUrl}`,
+    );
+
+    const expected = await this.findCatalogFiles(repoSlug);
+    const existing = await this.findExistingLocations(oldRepoUrl);
+
+    await this.updateForChanges(expected, existing);
+  }
+
+  private async updateForChanges(
+    expected: IngestionTarget[],
+    existing: LocationEntity[],
+  ): Promise<void> {
    const added: DeferredEntity[] = this.toDeferredEntities(
-      targets.filter(
+      expected.filter(
        // All Locations are managed by this provider and only have `target`, never `targets`.
        // All URLs (fileUrl, target) are created using `BitbucketCloudEntityProvider.toUrl`.
        // Hence, we can keep the comparison simple and don't need to handle different
@@ -265,7 +309,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
    const stillExisting: LocationEntity[] = [];
    const removed: DeferredEntity[] = [];
    existing.forEach(item => {
-      if (targets.find(value => value.fileUrl === item.spec.target)) {
+      if (expected.find(value => value.fileUrl === item.spec.target)) {
        stillExisting.push(item);
      } else {
        removed.push({
@@ -275,14 +319,17 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
      }
    });

-    const promises: Promise<void>[] = [
-      this.connection.refresh({
-        keys: stillExisting.map(entity => `url:${entity.spec.target}`),
-      }),
-    ];
+    const promises: Promise<void>[] =
+      stillExisting.length === 0
+        ? []
+        : [
+            this.connection!.refresh({
+              keys: stillExisting.map(entity => `url:${entity.spec.target}`),
+            }),
+          ];

    if (added.length > 0 || removed.length > 0) {
-      const connection = this.connection;
+      const connection = this.connection!;
      promises.push(
        connection.applyMutation({
          type: 'delta',