Merge pull request #5149 from goober/feature/bitbucket-discovery

Add a Bitbucket Discovery processor

.changeset/tricky-emus-wave.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add Bitbucket Server discovery processor.

docs/integrations/bitbucket/discovery.md

@@ -0,0 +1,41 @@
+---
+id: discovery
+title: Bitbucket Discovery
+sidebar_label: Discovery
+description:
+  Automatically discovering catalog entities from repositories in Bitbucket
+---
+
+The Bitbucket integration has a special discovery processor for discovering
+catalog entities located in Bitbucket. The processor will crawl your Bitbucket
+account and register entities matching the configured path. This can be useful
+as an alternative to static locations or manually adding things to the catalog.
+
+> Note: The Bitbucket Discovery Processor currently only supports a self-hosted
+> Bitbucket Server, and not the hosted Bitbucket Cloud product.
+
+To use the discovery processor, you'll need a Bitbucket integration
+[set up](locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`.
+Then you can add a location target to the catalog configuration:
+
+```yaml
+catalog:
+  locations:
+    - type: bitbucket-discovery
+      target: https://bitbucket.mycompany.com/projects/my-project/repos/service-*/catalog-info.yaml
+```
+
+Note the `bitbucket-discovery` type, as this is not a regular `url` processor.
+
+The target is composed of four parts:
+
+- The base instance URL, `https://bitbucket.mycompany.com` in this case
+- The project key to scan, which accepts \* wildcard tokens. This can simply be
+  `*` to scan repositories from all projects. This example only scans for
+  repositories in the `my-project` project.
+- The repository blob to scan, which accepts \* wildcard tokens. This can simply
+  be `*` to scan all repositories in the project. This example only looks for
+  repositories prefixed with `service-`.
+- The path within each repository to find the catalog YAML file. This will
+  usually be `/catalog-info.yaml` or a similar variation for catalog files
+  stored in the root directory of each repository.

docs/integrations/bitbucket/locations.md

@@ -1,12 +1,12 @@
 ---
 id: locations
-title: BitBucket Locations
+title: Bitbucket Locations
 sidebar_label: Locations
 description:
-  Integrating source code stored in BitBucket into the Backstage catalog
+  Integrating source code stored in Bitbucket into the Backstage catalog
 ---
 
-The BitBucket integration supports loading catalog entities from bitbucket.com
+The Bitbucket integration supports loading catalog entities from bitbucket.org
 or a self-hosted BitBucket. Entities can be added to
 [static catalog configuration](../../features/software-catalog/configuration.md),
 or registered with the
@@ -21,21 +21,21 @@ integrations:
       token: ${BITBUCKET_TOKEN}
 ```
 
-> Note: A public BitBucket provider is added automatically at startup for
+> Note: A public Bitbucket provider is added automatically at startup for
 > convenience, so you only need to list it if you want to supply a
 > [token](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html).
 
 Directly under the `bitbucket` key is a list of provider configurations, where
-you can list the BitBucket providers you want to fetch data from. Each entry is
+you can list the Bitbucket providers you want to fetch data from. Each entry is
 a structure with up to four elements:
 
-- `host`: The host of the BitBucket instance, e.g. `bitbucket.company.com`.
-- `token` (optional): An personal access token as expected by BitBucket. Either
+- `host`: The host of the Bitbucket instance, e.g. `bitbucket.company.com`.
+- `token` (optional): An personal access token as expected by Bitbucket. Either
   an access token **or** a username + appPassword may be supplied.
-- `username`: The BitBucket username to use in API requests. If neither a
+- `username`: The Bitbucket username to use in API requests. If neither a
   username nor token are supplied, anonymous access will be used.
-- `appPassword` (optional): The password for the BitBucket user. Only needed
+- `appPassword` (optional): The password for the Bitbucket user. Only needed
   when using `username` instead of `token`.
-- `apiBaseUrl` (optional): The URL of the GitLab API. For self-hosted
-  installations, it is commonly at `https://<host>/api/v4`. For gitlab.com, this
-  configuration is not needed as it can be inferred.
+- `apiBaseUrl` (optional): The URL of the Bitbucket API. For self-hosted
+  installations, it is commonly at `https://<host>/rest/api/1.0`. For
+  bitbucket.org, this configuration is not needed as it can be inferred.

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts

@@ -0,0 +1,218 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { getVoidLogger } from '@backstage/backend-common';
+import {
+  BitbucketDiscoveryProcessor,
+  readBitbucketOrg,
+} from './BitbucketDiscoveryProcessor';
+import { ConfigReader } from '@backstage/config';
+import { LocationSpec } from '@backstage/catalog-model';
+import { BitbucketClient, PagedResponse } from './bitbucket';
+
+function pagedResponse(values: any): PagedResponse<any> {
+  return {
+    values: values,
+    isLastPage: true,
+  } as PagedResponse<any>;
+}
+
+describe('BitbucketDiscoveryProcessor', () => {
+  const client: jest.Mocked<BitbucketClient> = {
+    listProjects: jest.fn(),
+    listRepositories: jest.fn(),
+  } as any;
+
+  afterEach(() => jest.resetAllMocks());
+
+  describe('reject unrelated entries', () => {
+    it('rejects unknown types', async () => {
+      const processor = BitbucketDiscoveryProcessor.fromConfig(
+        new ConfigReader({
+          integrations: {
+            bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
+          },
+        }),
+        { logger: getVoidLogger() },
+      );
+      const location: LocationSpec = {
+        type: 'not-bitbucket-discovery',
+        target: 'https://bitbucket.mycompany.com',
+      };
+      await expect(
+        processor.readLocation(location, false, () => {}),
+      ).resolves.toBeFalsy();
+    });
+
+    it('rejects unknown targets', async () => {
+      const processor = BitbucketDiscoveryProcessor.fromConfig(
+        new ConfigReader({
+          integrations: {
+            bitbucket: [
+              { host: 'bitbucket.org', token: 'blob' },
+              { host: 'bitbucket.mycompany.com', token: 'blob' },
+            ],
+          },
+        }),
+        { logger: getVoidLogger() },
+      );
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target: 'https://not.bitbucket.mycompany.com/foobar',
+      };
+      await expect(
+        processor.readLocation(location, false, () => {}),
+      ).rejects.toThrow(
+        /There is no Bitbucket integration that matches https:\/\/not.bitbucket.mycompany.com\/foobar/,
+      );
+    });
+  });
+
+  describe('handles repositories', () => {
+    it('output all repositories', async () => {
+      const target =
+        'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml';
+
+      client.listProjects.mockResolvedValue(
+        pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
+      );
+      client.listRepositories.mockResolvedValueOnce(
+        pagedResponse([
+          {
+            slug: 'backstage',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
+                },
+              ],
+            },
+          },
+        ]),
+      );
+      client.listRepositories.mockResolvedValueOnce(
+        pagedResponse([
+          {
+            slug: 'demo',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
+                },
+              ],
+            },
+          },
+        ]),
+      );
+
+      const actual = await readBitbucketOrg(client, target);
+      expect(actual.scanned).toBe(2);
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
+      });
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
+      });
+    });
+
+    it('output repositories with wildcards', async () => {
+      const target =
+        'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml';
+
+      client.listProjects.mockResolvedValue(
+        pagedResponse([{ key: 'backstage' }]),
+      );
+      client.listRepositories.mockResolvedValueOnce(
+        pagedResponse([
+          { slug: 'backstage' },
+          {
+            slug: 'techdocs-cli',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
+                },
+              ],
+            },
+          },
+          {
+            slug: 'techdocs-container',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
+                },
+              ],
+            },
+          },
+        ]),
+      );
+
+      const actual = await readBitbucketOrg(client, target);
+      expect(actual.scanned).toBe(3);
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
+      });
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
+      });
+    });
+    it('filter unrelated repositories', async () => {
+      const target =
+        'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml';
+
+      client.listProjects.mockResolvedValue(
+        pagedResponse([{ key: 'backstage' }]),
+      );
+      client.listRepositories.mockResolvedValue(
+        pagedResponse([
+          { slug: 'abstest' },
+          { slug: 'testxyz' },
+          {
+            slug: 'test',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/backstage/repos/test',
+                },
+              ],
+            },
+          },
+        ]),
+      );
+
+      const actual = await readBitbucketOrg(client, target);
+      expect(actual.scanned).toBe(3);
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
+      });
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts

@@ -0,0 +1,156 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Logger } from 'winston';
+import { Config } from '@backstage/config';
+
+import {
+  ScmIntegrationRegistry,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { LocationSpec } from '@backstage/catalog-model';
+import { BitbucketClient, paginated } from './bitbucket';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+import { results } from './index';
+
+export class BitbucketDiscoveryProcessor implements CatalogProcessor {
+  private readonly integrations: ScmIntegrationRegistry;
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const integrations = ScmIntegrations.fromConfig(config);
+
+    return new BitbucketDiscoveryProcessor({
+      ...options,
+      integrations,
+    });
+  }
+
+  constructor(options: {
+    integrations: ScmIntegrationRegistry;
+    logger: Logger;
+  }) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'bitbucket-discovery') {
+      return false;
+    }
+
+    const bitbucketConfig = this.integrations.bitbucket.byUrl(location.target)
+      ?.config;
+    if (!bitbucketConfig) {
+      throw new Error(
+        `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`,
+      );
+    } else if (bitbucketConfig.host === 'bitbucket.org') {
+      throw new Error(
+        `Component discovery for Bitbucket Cloud is not yet supported`,
+      );
+    }
+
+    const client = new BitbucketClient({
+      config: bitbucketConfig,
+    });
+    const startTimestamp = Date.now();
+    this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
+
+    const result = await readBitbucketOrg(client, location.target);
+
+    for (const repository of result.matches) {
+      emit(
+        results.location(
+          repository,
+          // Not all locations may actually exist, since the user defined them as a wildcard pattern.
+          // Thus, we emit them as optional and let the downstream processor find them while not outputting
+          // an error if it couldn't.
+          true,
+        ),
+      );
+    }
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`,
+    );
+
+    return true;
+  }
+}
+
+export async function readBitbucketOrg(
+  client: BitbucketClient,
+  target: string,
+): Promise<Result> {
+  const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target);
+  const projects = paginated(options => client.listProjects(options));
+  const result: Result = {
+    scanned: 0,
+    matches: [],
+  };
+
+  for await (const project of projects) {
+    if (!projectSearchPath.test(project.key)) {
+      continue;
+    }
+    const repositories = paginated(options =>
+      client.listRepositories(project.key, options),
+    );
+    for await (const repository of repositories) {
+      result.scanned++;
+
+      if (repoSearchPath.test(repository.slug)) {
+        result.matches.push({
+          type: 'url',
+          target: `${repository.links.self[0].href}${catalogPath}`,
+        });
+      }
+    }
+  }
+  return result;
+}
+
+function parseUrl(
+  urlString: string,
+): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } {
+  const url = new URL(urlString);
+  const path = url.pathname.substr(1).split('/');
+
+  // /projects/backstage/repos/techdocs-*/catalog-info.yaml
+  if (path.length > 3 && path[1].length && path[3].length) {
+    return {
+      projectSearchPath: escapeRegExp(decodeURIComponent(path[1])),
+      repoSearchPath: escapeRegExp(decodeURIComponent(path[3])),
+      catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`,
+    };
+  }
+
+  throw new Error(`Failed to parse ${urlString}`);
+}
+
+function escapeRegExp(str: string): RegExp {
+  return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
+}
+
+type Result = {
+  scanned: number;
+  matches: LocationSpec[];
+};

plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fetch from 'cross-fetch';
+
+import {
+  BitbucketIntegrationConfig,
+  getBitbucketRequestOptions,
+} from '@backstage/integration';
+
+export class BitbucketClient {
+  private readonly config: BitbucketIntegrationConfig;
+
+  constructor(options: { config: BitbucketIntegrationConfig }) {
+    this.config = options.config;
+  }
+
+  async listProjects(options?: ListOptions): Promise<PagedResponse<any>> {
+    return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options);
+  }
+
+  async listRepositories(
+    projectKey: string,
+    options?: ListOptions,
+  ): Promise<PagedResponse<any>> {
+    return this.pagedRequest(
+      `${this.config.apiBaseUrl}/projects/${projectKey}/repos`,
+      options,
+    );
+  }
+
+  private async pagedRequest(
+    endpoint: string,
+    options?: ListOptions,
+  ): Promise<PagedResponse<any>> {
+    const request = new URL(endpoint);
+    for (const key in options) {
+      if (options[key]) {
+        request.searchParams.append(key, options[key]!.toString());
+      }
+    }
+
+    const response = await fetch(
+      request.toString(),
+      getBitbucketRequestOptions(this.config),
+    );
+    if (!response.ok) {
+      throw new Error(
+        `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
+          response.status
+        } - ${response.statusText}`,
+      );
+    }
+    return response.json().then(repositories => {
+      return repositories as PagedResponse<any>;
+    });
+  }
+}
+
+export type ListOptions = {
+  [key: string]: number | undefined;
+  limit?: number | undefined;
+  start?: number | undefined;
+};
+
+export type PagedResponse<T> = {
+  size: number;
+  limit: number;
+  start: number;
+  isLastPage: boolean;
+  values: T[];
+  nextPageStart: number;
+};
+
+export async function* paginated(
+  request: (options: ListOptions) => Promise<PagedResponse<any>>,
+  options?: ListOptions,
+) {
+  const opts = options || { start: 0 };
+  let res;
+  do {
+    res = await request(opts);
+    opts.start = res.nextPageStart;
+    for (const item of res.values) {
+      yield item;
+    }
+  } while (!res.isLastPage);
+}

plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { BitbucketClient, paginated } from './client';
+export type { PagedResponse } from './client';

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -19,6 +19,7 @@ import * as results from './results';
 export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
 export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor';
 export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
+export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';
 export { FileReaderProcessor } from './FileReaderProcessor';

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -38,6 +38,7 @@ import {
 import { DatabaseManager } from '../database';
 import {
   AnnotateLocationEntityProcessor,
+  BitbucketDiscoveryProcessor,
   BuiltinKindsEntityProcessor,
   CatalogProcessor,
   CatalogProcessorParser,
@@ -304,6 +305,7 @@ export class CatalogBuilder {
     if (!this.processorsReplace) {
       processors.push(
         new FileReaderProcessor(),
+        BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
         GithubDiscoveryProcessor.fromConfig(config, { logger }),
         GithubOrgReaderProcessor.fromConfig(config, { logger }),
         LdapOrgReaderProcessor.fromConfig(config, { logger }),