Merge pull request #6615 from barticus/feature/bitbucket-cloud-discovery

Bitbucket cloud discovery support

.changeset/pretty-sloths-rush.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Bitbucket Cloud Discovery support

docs/integrations/bitbucket/discovery.md

@@ -11,12 +11,12 @@ catalog entities located in Bitbucket. The processor will crawl your Bitbucket
 account and register entities matching the configured path. This can be useful
 as an alternative to static locations or manually adding things to the catalog.
 
-> Note: The Bitbucket Discovery Processor currently only supports a self-hosted
-> Bitbucket Server, and not the hosted Bitbucket Cloud product.
+## Self-hosted Bitbucket Server
 
-To use the discovery processor, you'll need a Bitbucket integration
-[set up](locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`.
-Then you can add a location target to the catalog configuration:
+To use the discovery processor with a self-hosted Bitbucket Server, you'll need
+a Bitbucket integration [set up](locations.md) with a `BITBUCKET_TOKEN` and a
+`BITBUCKET_API_BASE_URL`. Then you can add a location target to the catalog
+configuration:
 
 ```yaml
 catalog:
@@ -44,6 +44,58 @@ The target is composed of four parts:
   will result in:
   `https://bitbucket.mycompany.com/projects/my-project/repos/service-a/catalog-info.yaml`.
 
+## Bitbucket Cloud
+
+To use the discovery processor with Bitbucket Cloud, you'll need a Bitbucket
+integration [set up](locations.md) with a `username` and an `appPassword`. Then
+you can add a location target to the catalog configuration:
+
+```yaml
+catalog:
+  locations:
+    - type: bitbucket-discovery
+      target: https://bitbucket.org/workspaces/my-workspace
+```
+
+Note the `bitbucket-discovery` type, as this is not a regular `url` processor.
+
+The target is composed of the following parts:
+
+- The base URL for Bitbucket, `https://bitbucket.org`
+- The workspace name to scan (following the `workspaces/` path part), which must
+  match a workspace accessible with the username of your integration.
+- (Optional) The project key to scan (following the `projects/` path part),
+  which accepts \* wildcard tokens. If omitted, repositories from all projects
+  in the workspace are included.
+- (Optional) The repository blob to scan (following the `repos/` path part),
+  which accepts \* wildcard tokens. If omitted, all repositories in the
+  workspace are included.
+- (Optional) The `catalogPath` query argument to specify the location within
+  each repository to find the catalog YAML file. This will usually be
+  `/catalog-info.yaml` or a similar variation for catalog files stored in the
+  root directory of each repository. If omitted, the default value
+  `catalog-info.yaml` will be used.
+- (Optional) The `q` query argument to be passed through to Bitbucket for
+  filtering results via the API. This is the most flexible option and will
+  reduce the amount of API calls if you have a large workspace.
+  [See here for the specification](https://developer.atlassian.com/bitbucket/api/2/reference/meta/filtering)
+  for the query argument (will be passed as the `q` query parameter).
+
+Examples:
+
+- `https://bitbucket.org/workspaces/my-workspace/projects/my-project` will find
+  all repositories in the `my-project` project in the `my-workspace` workspace.
+- `https://bitbucket.org/workspaces/my-workspace/repos/service-*` will find all
+  repositories starting with `service-` in the `my-workspace` workspace.
+- `https://bitbucket.org/workspaces/my-workspace/projects/apis-*/repos/service-*`
+  will find all repositories starting with `service-`, in all projects starting
+  with `apis-` in the `my-workspace` workspace.
+- `https://bitbucket.org/workspaces/my-workspace?q=project.key ~ "my-project"`
+  will find all repositories in a project containing `my-project` in its key.
+- `https://bitbucket.org/workspaces/my-workspace?catalogPath=my/nested/path/catalog.yaml`
+  will find all repositories in the `my-workspace` workspace and use the catalog
+  file at `my/nested/path/catalog.yaml`.
+
 ## Custom repository processing
 
 The Bitbucket Discovery Processor will by default emit a location for each

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts

@@ -17,7 +17,12 @@ import { getVoidLogger } from '@backstage/backend-common';
 import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 import { ConfigReader } from '@backstage/config';
 import { LocationSpec } from '@backstage/catalog-model';
-import { BitbucketRepositoryParser, PagedResponse } from './bitbucket';
+import {
+  BitbucketRepository20,
+  BitbucketRepositoryParser,
+  PagedResponse,
+  PagedResponse20,
+} from './bitbucket';
 import { results } from './index';
 import { RequestHandler, rest } from 'msw';
 import { setupServer } from 'msw/node';
@@ -79,6 +84,43 @@ function setupStubs(projects: any[]) {
   }
 }
 
+function setupBitbucketCloudStubs(
+  workspace: string,
+  repositories: Pick<BitbucketRepository20, 'slug' | 'project'>[],
+) {
+  const stubCallerFn = jest.fn();
+  function pagedResponse(values: any): PagedResponse20<any> {
+    return {
+      values: values,
+      page: 1,
+    } as PagedResponse20<any>;
+  }
+
+  server.use(
+    rest.get(
+      `https://api.bitbucket.org/2.0/repositories/${workspace}`,
+      (req, res, ctx) => {
+        stubCallerFn(req);
+        return res(
+          ctx.json(
+            pagedResponse(
+              repositories.map(r => ({
+                ...r,
+                links: {
+                  html: {
+                    href: `https://bitbucket.org/${workspace}/${r.slug}`,
+                  },
+                },
+              })),
+            ),
+          ),
+        );
+      },
+    ),
+  );
+  return stubCallerFn;
+}
+
 describe('BitbucketDiscoveryProcessor', () => {
   beforeAll(() => server.listen());
   afterEach(() => server.resetHandlers());
@@ -129,7 +171,7 @@ describe('BitbucketDiscoveryProcessor', () => {
     });
   });
 
-  describe('handles repositories', () => {
+  describe('handles organisation repositories', () => {
     const processor = BitbucketDiscoveryProcessor.fromConfig(
       new ConfigReader({
         integrations: {
@@ -255,6 +297,262 @@ describe('BitbucketDiscoveryProcessor', () => {
     });
   });
 
+  describe('handles cloud repositories', () => {
+    const processor = BitbucketDiscoveryProcessor.fromConfig(
+      new ConfigReader({
+        integrations: {
+          bitbucket: [
+            {
+              host: 'bitbucket.org',
+              username: 'myuser',
+              appPassword: 'blob',
+            },
+          ],
+        },
+      }),
+      { logger: getVoidLogger() },
+    );
+
+    it('output all repositories by default', async () => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+        { project: { key: 'prj-two' }, slug: 'repository-two' },
+      ]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target: 'https://bitbucket.org/workspaces/myworkspace',
+      };
+
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toBeCalledTimes(2);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
+        },
+        optional: true,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-two/src/master/catalog-info.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it('uses provided catalog path', async () => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+        { project: { key: 'prj-two' }, slug: 'repository-two' },
+      ]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.org/workspaces/myworkspace?catalogPath=my/nested/path/catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toBeCalledTimes(2);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-one/src/master/my/nested/path/catalog.yaml',
+        },
+        optional: true,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-two/src/master/my/nested/path/catalog.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it('output all repositories', async () => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+        { project: { key: 'prj-two' }, slug: 'repository-two' },
+      ]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*?catalogPath=catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toBeCalledTimes(2);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
+        },
+        optional: true,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-two/src/master/catalog.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it('output repositories with wildcards', async () => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+        { project: { key: 'prj-two' }, slug: 'repository-two' },
+      ]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*?catalogPath=catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toBeCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it('filter unrelated repositories', async () => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+        { project: { key: 'prj-one' }, slug: 'repository-two' },
+        { project: { key: 'prj-one' }, slug: 'repository-three' },
+      ]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three?catalogPath=catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toBeCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-three/src/master/catalog.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it('submits query', async () => {
+      const mockCall = setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+      ]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.org/workspaces/myworkspace?q=project.key ~ "prj-one"',
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toBeCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
+        },
+        optional: true,
+      });
+      expect(mockCall).toBeCalledTimes(1);
+      // it should be possible to do this via an `expect.objectContaining` check but seems to fail with some encoding issue.
+      expect(mockCall.mock.calls[0][0].url).toMatchInlineSnapshot(
+        `"https://api.bitbucket.org/2.0/repositories/myworkspace?page=1&pagelen=100&q=project.key+%7E+%22prj-one%22"`,
+      );
+    });
+
+    it.each`
+      target
+      ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*'}
+      ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/'}
+      ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-*/'}
+    `("target '$target' adds default path to catalog", async ({ target }) => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+      ]);
+
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target: target,
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it.each`
+      target
+      ${'https://bitbucket.org/test'}
+    `("target '$target' is rejected", async ({ target }) => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+      ]);
+
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target: target,
+      };
+
+      const emitter = jest.fn();
+      await expect(
+        processor.readLocation(location, false, emitter),
+      ).rejects.toThrow(/Failed to parse /);
+    });
+  });
+
   describe('Custom repository parser', () => {
     const customRepositoryParser: BitbucketRepositoryParser =
       async function* customRepositoryParser({}) {

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts

@@ -17,6 +17,7 @@ import { Logger } from 'winston';
 import { Config } from '@backstage/config';
 
 import {
+  BitbucketIntegration,
   ScmIntegrationRegistry,
   ScmIntegrations,
 } from '@backstage/integration';
@@ -26,10 +27,16 @@ import {
   BitbucketClient,
   defaultRepositoryParser,
   paginated,
+  paginated20,
   BitbucketRepository,
+  BitbucketRepository20,
 } from './bitbucket';
 import { CatalogProcessor, CatalogProcessorEmit } from './types';
 
+const DEFAULT_BRANCH = 'master';
+const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml';
+const EMPTY_CATALOG_LOCATION = '/';
+
 export class BitbucketDiscoveryProcessor implements CatalogProcessor {
   private readonly integrations: ScmIntegrationRegistry;
   private readonly parser: BitbucketRepositoryParser;
@@ -71,10 +78,6 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
       throw new Error(
         `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`,
       );
-    } else if (integration.config.host === 'bitbucket.org') {
-      throw new Error(
-        `Component discovery for Bitbucket Cloud is not yet supported`,
-      );
     }
 
     const client = new BitbucketClient({
@@ -83,38 +86,89 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
     const startTimestamp = Date.now();
     this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
 
-    const { catalogPath } = parseUrl(location.target);
-    const expandedCatalogPath =
-      catalogPath === '/' ? '/catalog-info.yaml' : catalogPath;
+    const isBitbucketCloud = integration.config.host === 'bitbucket.org';
 
-    const result = await readBitbucketOrg(client, location.target);
+    const processOptions: ProcessOptions = {
+      client,
+      emit,
+      integration,
+      location,
+    };
 
+    const { scanned, matches } = isBitbucketCloud
+      ? await this.processCloudRepositories(processOptions)
+      : await this.processOrganisationRepositories(processOptions);
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${scanned} Bitbucket repositories (${matches} matching the pattern) in ${duration} seconds`,
+    );
+
+    return true;
+  }
+
+  private async processCloudRepositories(
+    options: ProcessOptions,
+  ): Promise<ResultSummary> {
+    const { client, location, integration, emit } = options;
+
+    const { catalogPath: requestedCatalogPath } = parseBitbucketCloudUrl(
+      location.target,
+    );
+    const catalogPath =
+      requestedCatalogPath === EMPTY_CATALOG_LOCATION
+        ? DEFAULT_CATALOG_LOCATION
+        : requestedCatalogPath;
+    const result = await readBitbucketCloud(client, location.target);
     for (const repository of result.matches) {
+      const mainbranch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
       for await (const entity of this.parser({
-        integration: integration,
-        target: `${repository.links.self[0].href}${expandedCatalogPath}`,
+        integration,
+        target: `${repository.links.html.href}/src/${mainbranch}${catalogPath}`,
         logger: this.logger,
       })) {
         emit(entity);
       }
     }
+    return {
+      matches: result.matches.length,
+      scanned: result.scanned,
+    };
+  }
 
-    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
-    this.logger.debug(
-      `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`,
-    );
-
-    return true;
+  private async processOrganisationRepositories(
+    options: ProcessOptions,
+  ): Promise<ResultSummary> {
+    const { client, location, integration, emit } = options;
+    const { catalogPath: requestedCatalogPath } = parseUrl(location.target);
+    const catalogPath =
+      requestedCatalogPath === EMPTY_CATALOG_LOCATION
+        ? DEFAULT_CATALOG_LOCATION
+        : requestedCatalogPath;
+    const result = await readBitbucketOrg(client, location.target);
+    for (const repository of result.matches) {
+      for await (const entity of this.parser({
+        integration,
+        target: `${repository.links.self[0].href}${catalogPath}`,
+        logger: this.logger,
+      })) {
+        emit(entity);
+      }
+    }
+    return {
+      matches: result.matches.length,
+      scanned: result.scanned,
+    };
   }
 }
 
 export async function readBitbucketOrg(
   client: BitbucketClient,
   target: string,
-): Promise<Result> {
+): Promise<Result<BitbucketRepository>> {
   const { projectSearchPath, repoSearchPath } = parseUrl(target);
   const projects = paginated(options => client.listProjects(options));
-  const result: Result = {
+  const result: Result<BitbucketRepository> = {
     scanned: 0,
     matches: [],
   };
@@ -136,6 +190,40 @@ export async function readBitbucketOrg(
   return result;
 }
 
+export async function readBitbucketCloud(
+  client: BitbucketClient,
+  target: string,
+): Promise<Result<BitbucketRepository20>> {
+  const {
+    workspacePath,
+    queryParam: q,
+    projectSearchPath,
+    repoSearchPath,
+  } = parseBitbucketCloudUrl(target);
+
+  const repositories = paginated20(
+    options => client.listRepositoriesByWorkspace20(workspacePath, options),
+    {
+      q,
+    },
+  );
+  const result: Result<BitbucketRepository20> = {
+    scanned: 0,
+    matches: [],
+  };
+
+  for await (const repository of repositories) {
+    result.scanned++;
+    if (
+      (!projectSearchPath || projectSearchPath.test(repository.project.key)) &&
+      (!repoSearchPath || repoSearchPath.test(repository.slug))
+    ) {
+      result.matches.push(repository);
+    }
+  }
+  return result;
+}
+
 function parseUrl(urlString: string): {
   projectSearchPath: RegExp;
   repoSearchPath: RegExp;
@@ -156,11 +244,59 @@ function parseUrl(urlString: string): {
   throw new Error(`Failed to parse ${urlString}`);
 }
 
+function readPathParameters(pathParts: string[]): Map<string, string> {
+  const vals: Record<string, any> = {};
+  for (let i = 0; i + 1 < pathParts.length; i += 2) {
+    vals[pathParts[i]] = decodeURIComponent(pathParts[i + 1]);
+  }
+  return new Map<string, string>(Object.entries(vals));
+}
+
+function parseBitbucketCloudUrl(urlString: string): {
+  workspacePath: string;
+  catalogPath: string;
+  projectSearchPath?: RegExp;
+  repoSearchPath?: RegExp;
+  queryParam?: string;
+} {
+  const url = new URL(urlString);
+  const pathMap = readPathParameters(url.pathname.substr(1).split('/'));
+  const query = url.searchParams;
+
+  if (!pathMap.has('workspaces')) {
+    throw new Error(`Failed to parse workspace from ${urlString}`);
+  }
+
+  return {
+    workspacePath: pathMap.get('workspaces')!,
+    projectSearchPath: pathMap.has('projects')
+      ? escapeRegExp(pathMap.get('projects')!)
+      : undefined,
+    repoSearchPath: pathMap.has('repos')
+      ? escapeRegExp(pathMap.get('repos')!)
+      : undefined,
+    catalogPath: `/${query.get('catalogPath') || ''}`,
+    queryParam: query.get('q') || undefined,
+  };
+}
+
 function escapeRegExp(str: string): RegExp {
   return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
 }
 
-type Result = {
-  scanned: number;
-  matches: BitbucketRepository[];
+type ProcessOptions = {
+  client: BitbucketClient;
+  integration: BitbucketIntegration;
+  location: LocationSpec;
+  emit: CatalogProcessorEmit;
+};
+
+type Result<T> = {
+  scanned: number;
+  matches: T[];
+};
+
+type ResultSummary = {
+  scanned: number;
+  matches: number;
 };

plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts

@@ -19,6 +19,7 @@ import {
   BitbucketIntegrationConfig,
   getBitbucketRequestOptions,
 } from '@backstage/integration';
+import { BitbucketRepository20 } from './types';
 
 export class BitbucketClient {
   private readonly config: BitbucketIntegrationConfig;
@@ -31,12 +32,24 @@ export class BitbucketClient {
     return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options);
   }
 
+  async listRepositoriesByWorkspace20(
+    workspace: string,
+    options?: ListOptions20,
+  ): Promise<PagedResponse20<BitbucketRepository20>> {
+    return this.pagedRequest20<BitbucketRepository20>(
+      `${this.config.apiBaseUrl}/repositories/${encodeURIComponent(workspace)}`,
+      options,
+    );
+  }
+
   async listRepositories(
     projectKey: string,
     options?: ListOptions,
   ): Promise<PagedResponse<any>> {
     return this.pagedRequest(
-      `${this.config.apiBaseUrl}/projects/${projectKey}/repos`,
+      `${this.config.apiBaseUrl}/projects/${encodeURIComponent(
+        projectKey,
+      )}/repos`,
       options,
     );
   }
@@ -63,9 +76,32 @@ export class BitbucketClient {
         } - ${response.statusText}`,
       );
     }
-    return response.json().then(repositories => {
-      return repositories as PagedResponse<any>;
-    });
+    return response.json() as Promise<PagedResponse<any>>;
+  }
+
+  private async pagedRequest20<T = any>(
+    endpoint: string,
+    options?: ListOptions20,
+  ): Promise<PagedResponse20<T>> {
+    const request = new URL(endpoint);
+    for (const key in options) {
+      if (options[key]) {
+        request.searchParams.append(key, options[key]!.toString());
+      }
+    }
+
+    const response = await fetch(
+      request.toString(),
+      getBitbucketRequestOptions(this.config),
+    );
+    if (!response.ok) {
+      throw new Error(
+        `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
+          response.status
+        } - ${response.statusText}`,
+      );
+    }
+    return response.json() as Promise<PagedResponse20<T>>;
   }
 }
 
@@ -84,6 +120,20 @@ export type PagedResponse<T> = {
   nextPageStart: number;
 };
 
+export type ListOptions20 = {
+  [key: string]: string | number | undefined;
+  page?: number | undefined;
+  pagelen?: number | undefined;
+};
+
+export type PagedResponse20<T> = {
+  page: number;
+  pagelen: number;
+  size: number;
+  values: T[];
+  next: string;
+};
+
 export async function* paginated(
   request: (options: ListOptions) => Promise<PagedResponse<any>>,
   options?: ListOptions,
@@ -98,3 +148,18 @@ export async function* paginated(
     }
   } while (!res.isLastPage);
 }
+
+export async function* paginated20<T = any>(
+  request: (options: ListOptions20) => Promise<PagedResponse20<T>>,
+  options?: ListOptions20,
+) {
+  const opts = { page: 1, pagelen: 100, ...options };
+  let res;
+  do {
+    res = await request(opts);
+    opts.page = opts.page + 1;
+    for (const item of res.values) {
+      yield item;
+    }
+  } while (res.next);
+}

plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts

@@ -13,8 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export { BitbucketClient, paginated } from './client';
+export { BitbucketClient, paginated, paginated20 } from './client';
 export { defaultRepositoryParser } from './BitbucketRepositoryParser';
-export type { PagedResponse } from './client';
-export type { BitbucketRepository } from './types';
+export type { PagedResponse, PagedResponse20 } from './client';
+export type { BitbucketRepository, BitbucketRepository20 } from './types';
 export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser';

plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts

@@ -13,11 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export type BitbucketRepository = {
+
+type BitbucketRepositoryBase = {
   project: {
     key: string;
   };
   slug: string;
+};
+
+export type BitbucketRepository = BitbucketRepositoryBase & {
   links: Record<
     string,
     {
@@ -25,3 +29,33 @@ export type BitbucketRepository = {
     }[]
   >;
 };
+
+export type BitbucketRepository20 = BitbucketRepositoryBase & {
+  links: Record<
+    | 'self'
+    | 'source'
+    | 'html'
+    | 'avatar'
+    | 'pullrequests'
+    | 'commits'
+    | 'forks'
+    | 'watchers'
+    | 'downloads'
+    | 'hooks',
+    {
+      href: string;
+      name?: string;
+    }
+  > &
+    Record<
+      'clone',
+      {
+        href: string;
+        name?: string;
+      }[]
+    >;
+  mainbranch?: {
+    type: string;
+    name: string;
+  };
+};