Bitbucket Cloud Discovery

When bitbucket.org is discovered as the target for the bitbucket-discovery catalog processor, a scan is performed in a workspace and finds repositories matching the any project and repository regexes in line with the existing bitbucket server implementation. Resolves #6216. Signed-off-by: Bart Breen <bart.breen@twobulls.com>
2021-07-27 11:15:49 +10:00
parent 71b5cc1ba8
commit e2ed9126e7
6 changed files with 462 additions and 32 deletions
@@ -11,12 +11,12 @@ catalog entities located in Bitbucket. The processor will crawl your Bitbucket
 account and register entities matching the configured path. This can be useful
 as an alternative to static locations or manually adding things to the catalog.

-> Note: The Bitbucket Discovery Processor currently only supports a self-hosted
-> Bitbucket Server, and not the hosted Bitbucket Cloud product.
+## Self-hosted Bitbucket Server

-To use the discovery processor, you'll need a Bitbucket integration
-[set up](locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`.
-Then you can add a location target to the catalog configuration:
+To use the discovery processor with a self-hosted Bitbucket Server, you'll need
+a Bitbucket integration [set up](locations.md) with a `BITBUCKET_TOKEN` and a
+`BITBUCKET_API_BASE_URL`. Then you can add a location target to the catalog
+configuration:

 ```yaml
 catalog:
@@ -44,6 +44,41 @@ The target is composed of four parts:
  will result in:
  `https://bitbucket.mycompany.com/projects/my-project/repos/service-a/catalog-info.yaml`.

+## Bitbucket Cloud
+
+To use the discovery processor with Bitbucket Cloud, you'll need a Bitbucket
+integration [set up](locations.md) with a `username` and an `appPassword`. Then
+you can add a location target to the catalog configuration:
+
+```yaml
+catalog:
+  locations:
+    - type: bitbucket-discovery
+      target: https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/catalog-info.yaml
+```
+
+Note the `bitbucket-discovery` type, as this is not a regular `url` processor.
+
+The target is composed of four parts:
+
+- The base instance URL, `https://bitbucket.org` in this case
+- The workspace name to scan, which must match a workspace accessible with the
+  username of your integration.
+- The project key to scan, which accepts \* wildcard tokens. This can simply be
+  `*` to include all repositories. This example only returns repositories in the
+  `my-project` project.
+- The repository blob to scan, which accepts \* wildcard tokens. This can simply
+  be `*` to scan all repositories in the workspace. This example only looks for
+  repositories prefixed with `service-`.
+- The path within each repository to find the catalog YAML file. This will
+  usually be `/catalog-info.yaml` or a similar variation for catalog files
+  stored in the root directory of each repository. If omitted, the default value
+  `catalog-info.yaml` will be used. E.g. given that `my-project` and `service-a`
+  exists,
+  `https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/`
+  will result in:
+  `https://bitbucket.org/my-workspace/service-a/src/master/catalog-info.yaml`.
+
 ## Custom repository processing

 The Bitbucket Discovery Processor will by default emit a location for each
@@ -17,7 +17,13 @@ import { getVoidLogger } from '@backstage/backend-common';
 import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 import { ConfigReader } from '@backstage/config';
 import { LocationSpec } from '@backstage/catalog-model';
-import { BitbucketRepositoryParser, PagedResponse } from './bitbucket';
+import {
+  BitbucketRepository,
+  BitbucketRepository20,
+  BitbucketRepositoryParser,
+  PagedResponse,
+  PagedResponse20,
+} from './bitbucket';
 import { results } from './index';
 import { RequestHandler, rest } from 'msw';
 import { setupServer } from 'msw/node';
@@ -79,6 +85,40 @@ function setupStubs(projects: any[]) {
  }
 }

+function setupBitbucketCloudStubs(
+  workspace: string,
+  repositories: Pick<BitbucketRepository20, 'slug' | 'project'>[],
+) {
+  function pagedResponse(values: any): PagedResponse20<any> {
+    return {
+      values: values,
+      page: 1,
+    } as PagedResponse20<any>;
+  }
+
+  server.use(
+    rest.get(
+      `https://api.bitbucket.org/2.0/repositories/${workspace}`,
+      (_, res, ctx) => {
+        return res(
+          ctx.json(
+            pagedResponse(
+              repositories.map(r => ({
+                ...r,
+                links: {
+                  source: {
+                    href: `https://api.bitbucket.org/2.0/repositories/${workspace}/${r.slug}/src`,
+                  },
+                },
+              })),
+            ),
+          ),
+        );
+      },
+    ),
+  );
+}
+
 describe('BitbucketDiscoveryProcessor', () => {
  beforeAll(() => server.listen());
  afterEach(() => server.resetHandlers());
@@ -129,7 +169,7 @@ describe('BitbucketDiscoveryProcessor', () => {
    });
  });

-  describe('handles repositories', () => {
+  describe('handles organisation repositories', () => {
    const processor = BitbucketDiscoveryProcessor.fromConfig(
      new ConfigReader({
        integrations: {
@@ -255,6 +295,142 @@ describe('BitbucketDiscoveryProcessor', () => {
    });
  });

+  describe('handles cloud repositories', () => {
+    const processor = BitbucketDiscoveryProcessor.fromConfig(
+      new ConfigReader({
+        integrations: {
+          bitbucket: [
+            {
+              host: 'bitbucket.org',
+              username: 'myuser',
+              appPassword: 'blob',
+            },
+          ],
+        },
+      }),
+      { logger: getVoidLogger() },
+    );
+
+    it('output all repositories', async () => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+        { project: { key: 'prj-two' }, slug: 'repository-two' },
+      ]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*/catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toBeCalledTimes(2);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml',
+        },
+        optional: true,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-two/src/master/catalog.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it('output repositories with wildcards', async () => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+        { project: { key: 'prj-two' }, slug: 'repository-two' },
+      ]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toBeCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml',
+        },
+        optional: true,
+      });
+    });
+    it('filter unrelated repositories', async () => {
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+        { project: { key: 'prj-one' }, slug: 'repository-two' },
+        { project: { key: 'prj-one' }, slug: 'repository-three' },
+      ]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three/catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toBeCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-three/src/master/catalog.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it.each`
+      target
+      ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*'}
+      ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/'}
+      ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-*/'}
+    `("target '$target' adds default path to catalog", async ({ target }) => {
+      setupStubs([{ key: 'backstage', repos: ['techdocs-cli'] }]);
+      setupBitbucketCloudStubs('myworkspace', [
+        { project: { key: 'prj-one' }, slug: 'repository-one' },
+      ]);
+
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target: target,
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog-info.yaml',
+        },
+        optional: true,
+      });
+    });
+  });
+
  describe('Custom repository parser', () => {
    const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({}) {
      yield results.location(
@@ -17,6 +17,7 @@ import { Logger } from 'winston';
 import { Config } from '@backstage/config';

 import {
+  BitbucketIntegration,
  ScmIntegrationRegistry,
  ScmIntegrations,
 } from '@backstage/integration';
@@ -26,10 +27,16 @@ import {
  BitbucketClient,
  defaultRepositoryParser,
  paginated,
+  paginated20,
  BitbucketRepository,
+  BitbucketRepository20,
 } from './bitbucket';
 import { CatalogProcessor, CatalogProcessorEmit } from './types';

+const DEFAULT_BRANCH = 'master';
+const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml';
+const EMPTY_CATALOG_LOCATION = '/';
+
 export class BitbucketDiscoveryProcessor implements CatalogProcessor {
  private readonly integrations: ScmIntegrationRegistry;
  private readonly parser: BitbucketRepositoryParser;
@@ -71,10 +78,6 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
      throw new Error(
        `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`,
      );
-    } else if (integration.config.host === 'bitbucket.org') {
-      throw new Error(
-        `Component discovery for Bitbucket Cloud is not yet supported`,
-      );
    }

    const client = new BitbucketClient({
@@ -83,38 +86,89 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
    const startTimestamp = Date.now();
    this.logger.info(`Reading Bitbucket repositories from ${location.target}`);

-    const { catalogPath } = parseUrl(location.target);
-    const expandedCatalogPath =
-      catalogPath === '/' ? '/catalog-info.yaml' : catalogPath;
+    const isBitbucketCloud = integration.config.host === 'bitbucket.org';

-    const result = await readBitbucketOrg(client, location.target);
+    const processOptions: ProcessOptions = {
+      client,
+      emit,
+      integration,
+      location,
+    };

+    const { scanned, matches } = isBitbucketCloud
+      ? await this.processCloudRepositories(processOptions)
+      : await this.processOrganisationRepositories(processOptions);
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${scanned} Bitbucket repositories (${matches} matching the pattern) in ${duration} seconds`,
+    );
+
+    return true;
+  }
+
+  private async processCloudRepositories(
+    options: ProcessOptions,
+  ): Promise<ResultSummary> {
+    const { client, location, integration, emit } = options;
+
+    const { catalogPath: requestedCatalogPath } = parseBitbucketCloudUrl(
+      location.target,
+    );
+    const catalogPath =
+      requestedCatalogPath === EMPTY_CATALOG_LOCATION
+        ? DEFAULT_CATALOG_LOCATION
+        : requestedCatalogPath;
+    const result = await readBitbucketCloud(client, location.target);
    for (const repository of result.matches) {
+      const mainbranch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
      for await (const entity of this.parser({
-        integration: integration,
-        target: `${repository.links.self[0].href}${expandedCatalogPath}`,
+        integration,
+        target: `${repository.links.source.href}/${mainbranch}${catalogPath}`,
        logger: this.logger,
      })) {
        emit(entity);
      }
    }
+    return {
+      matches: result.matches.length,
+      scanned: result.scanned,
+    };
+  }

-    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
-    this.logger.debug(
-      `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`,
-    );
-
-    return true;
+  private async processOrganisationRepositories(
+    options: ProcessOptions,
+  ): Promise<ResultSummary> {
+    const { client, location, integration, emit } = options;
+    const { catalogPath: requestedCatalogPath } = parseUrl(location.target);
+    const catalogPath =
+      requestedCatalogPath === EMPTY_CATALOG_LOCATION
+        ? DEFAULT_CATALOG_LOCATION
+        : requestedCatalogPath;
+    const result = await readBitbucketOrg(client, location.target);
+    for (const repository of result.matches) {
+      for await (const entity of this.parser({
+        integration,
+        target: `${repository.links.self[0].href}${catalogPath}`,
+        logger: this.logger,
+      })) {
+        emit(entity);
+      }
+    }
+    return {
+      matches: result.matches.length,
+      scanned: result.scanned,
+    };
  }
 }

 export async function readBitbucketOrg(
  client: BitbucketClient,
  target: string,
-): Promise<Result> {
+): Promise<Result<BitbucketRepository>> {
  const { projectSearchPath, repoSearchPath } = parseUrl(target);
  const projects = paginated(options => client.listProjects(options));
-  const result: Result = {
+  const result: Result<BitbucketRepository> = {
    scanned: 0,
    matches: [],
  };
@@ -136,6 +190,35 @@ export async function readBitbucketOrg(
  return result;
 }

+export async function readBitbucketCloud(
+  client: BitbucketClient,
+  target: string,
+): Promise<Result<BitbucketRepository20>> {
+  const {
+    workspacePath,
+    projectSearchPath,
+    repoSearchPath,
+  } = parseBitbucketCloudUrl(target);
+  const repositories = paginated20(options =>
+    client.listRepositoriesByWorkspace20(workspacePath, options),
+  );
+  const result: Result<BitbucketRepository20> = {
+    scanned: 0,
+    matches: [],
+  };
+
+  for await (const repository of repositories) {
+    result.scanned++;
+    if (
+      projectSearchPath.test(repository.project.key) &&
+      repoSearchPath.test(repository.slug)
+    ) {
+      result.matches.push(repository);
+    }
+  }
+  return result;
+}
+
 function parseUrl(
  urlString: string,
 ): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } {
@@ -154,11 +237,47 @@ function parseUrl(
  throw new Error(`Failed to parse ${urlString}`);
 }

+function parseBitbucketCloudUrl(
+  urlString: string,
+): {
+  workspacePath: string;
+  projectSearchPath: RegExp;
+  repoSearchPath: RegExp;
+  catalogPath: string;
+} {
+  const url = new URL(urlString);
+  const path = url.pathname.substr(1).split('/');
+
+  // workspaces/{workspacePath}/projects/{projectSearchPath}/repos/{repoSearchPath}/{catalogPath=catalog-info.yaml}
+  if (path.length > 5 && path[1].length && path[3].length) {
+    return {
+      workspacePath: decodeURIComponent(path[1]),
+      projectSearchPath: escapeRegExp(decodeURIComponent(path[3])),
+      repoSearchPath: escapeRegExp(decodeURIComponent(path[5])),
+      catalogPath: `/${decodeURIComponent(path.slice(6).join('/'))}`,
+    };
+  }
+
+  throw new Error(`Failed to parse ${urlString}`);
+}
+
 function escapeRegExp(str: string): RegExp {
  return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
 }

-type Result = {
-  scanned: number;
-  matches: BitbucketRepository[];
+type ProcessOptions = {
+  client: BitbucketClient;
+  integration: BitbucketIntegration;
+  location: LocationSpec;
+  emit: CatalogProcessorEmit;
+};
+
+type Result<T> = {
+  scanned: number;
+  matches: T[];
+};
+
+type ResultSummary = {
+  scanned: number;
+  matches: number;
 };
@@ -19,6 +19,7 @@ import {
  BitbucketIntegrationConfig,
  getBitbucketRequestOptions,
 } from '@backstage/integration';
+import { BitbucketRepository, BitbucketRepository20 } from './types';

 export class BitbucketClient {
  private readonly config: BitbucketIntegrationConfig;
@@ -31,6 +32,16 @@ export class BitbucketClient {
    return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options);
  }

+  async listRepositoriesByWorkspace20(
+    workspace: string,
+    options?: ListOptions,
+  ): Promise<PagedResponse20<BitbucketRepository20>> {
+    return this.pagedRequest20<BitbucketRepository20>(
+      `${this.config.apiBaseUrl}/repositories/${workspace}`,
+      options,
+    );
+  }
+
  async listRepositories(
    projectKey: string,
    options?: ListOptions,
@@ -67,6 +78,33 @@ export class BitbucketClient {
      return repositories as PagedResponse<any>;
    });
  }
+
+  private async pagedRequest20<T = any>(
+    endpoint: string,
+    options?: ListOptions,
+  ): Promise<PagedResponse20<T>> {
+    const request = new URL(endpoint);
+    for (const key in options) {
+      if (options[key]) {
+        request.searchParams.append(key, options[key]!.toString());
+      }
+    }
+
+    const response = await fetch(
+      request.toString(),
+      getBitbucketRequestOptions(this.config),
+    );
+    if (!response.ok) {
+      throw new Error(
+        `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
+          response.status
+        } - ${response.statusText}`,
+      );
+    }
+    return response.json().then(repositories => {
+      return repositories as PagedResponse20<T>;
+    });
+  }
 }

 export type ListOptions = {
@@ -84,6 +122,19 @@ export type PagedResponse<T> = {
  nextPageStart: number;
 };

+export type ListOptions20 = {
+  [key: string]: number | undefined;
+  page?: number | undefined;
+};
+
+export type PagedResponse20<T> = {
+  page: number;
+  pagelen: number;
+  size: number;
+  values: T[];
+  next: string;
+};
+
 export async function* paginated(
  request: (options: ListOptions) => Promise<PagedResponse<any>>,
  options?: ListOptions,
@@ -98,3 +149,18 @@ export async function* paginated(
    }
  } while (!res.isLastPage);
 }
+
+export async function* paginated20<T = any>(
+  request: (options: ListOptions20) => Promise<PagedResponse20<T>>,
+  options?: ListOptions20,
+) {
+  const opts = options || { page: 1 };
+  let res;
+  do {
+    res = await request(opts);
+    opts.page = (opts.page || 1) + 1;
+    for (const item of res.values) {
+      yield item;
+    }
+  } while (res.next);
+}
@@ -13,8 +13,8 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-export { BitbucketClient, paginated } from './client';
+export { BitbucketClient, paginated, paginated20 } from './client';
 export { defaultRepositoryParser } from './BitbucketRepositoryParser';
-export type { PagedResponse } from './client';
-export type { BitbucketRepository } from './types';
+export type { PagedResponse, PagedResponse20 } from './client';
+export type { BitbucketRepository, BitbucketRepository20 } from './types';
 export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser';
@@ -13,11 +13,15 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-export type BitbucketRepository = {
+
+type BitbucketRepositoryBase = {
  project: {
    key: string;
  };
  slug: string;
+};
+
+export type BitbucketRepository = BitbucketRepositoryBase & {
  links: Record<
    string,
    {
@@ -25,3 +29,33 @@ export type BitbucketRepository = {
    }[]
  >;
 };
+
+export type BitbucketRepository20 = BitbucketRepositoryBase & {
+  links: Record<
+    | 'self'
+    | 'source'
+    | 'html'
+    | 'avatar'
+    | 'pullrequests'
+    | 'commits'
+    | 'forks'
+    | 'watchers'
+    | 'downloads'
+    | 'hooks',
+    {
+      href: string;
+      name?: string;
+    }
+  > &
+    Record<
+      'clone',
+      {
+        href: string;
+        name?: string;
+      }[]
+    >;
+  mainbranch?: {
+    type: string;
+    name: string;
+  };
+};