Merge pull request #5149 from goober/feature/bitbucket-discovery

Add a Bitbucket Discovery processor
2021-03-30 14:40:52 +02:00
parent bd90c8f58a 13abbc15da
commit aa618e57bb
9 changed files with 552 additions and 12 deletions
@@ -0,0 +1,218 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { getVoidLogger } from '@backstage/backend-common';
+import {
+  BitbucketDiscoveryProcessor,
+  readBitbucketOrg,
+} from './BitbucketDiscoveryProcessor';
+import { ConfigReader } from '@backstage/config';
+import { LocationSpec } from '@backstage/catalog-model';
+import { BitbucketClient, PagedResponse } from './bitbucket';
+
+function pagedResponse(values: any): PagedResponse<any> {
+  return {
+    values: values,
+    isLastPage: true,
+  } as PagedResponse<any>;
+}
+
+describe('BitbucketDiscoveryProcessor', () => {
+  const client: jest.Mocked<BitbucketClient> = {
+    listProjects: jest.fn(),
+    listRepositories: jest.fn(),
+  } as any;
+
+  afterEach(() => jest.resetAllMocks());
+
+  describe('reject unrelated entries', () => {
+    it('rejects unknown types', async () => {
+      const processor = BitbucketDiscoveryProcessor.fromConfig(
+        new ConfigReader({
+          integrations: {
+            bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
+          },
+        }),
+        { logger: getVoidLogger() },
+      );
+      const location: LocationSpec = {
+        type: 'not-bitbucket-discovery',
+        target: 'https://bitbucket.mycompany.com',
+      };
+      await expect(
+        processor.readLocation(location, false, () => {}),
+      ).resolves.toBeFalsy();
+    });
+
+    it('rejects unknown targets', async () => {
+      const processor = BitbucketDiscoveryProcessor.fromConfig(
+        new ConfigReader({
+          integrations: {
+            bitbucket: [
+              { host: 'bitbucket.org', token: 'blob' },
+              { host: 'bitbucket.mycompany.com', token: 'blob' },
+            ],
+          },
+        }),
+        { logger: getVoidLogger() },
+      );
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target: 'https://not.bitbucket.mycompany.com/foobar',
+      };
+      await expect(
+        processor.readLocation(location, false, () => {}),
+      ).rejects.toThrow(
+        /There is no Bitbucket integration that matches https:\/\/not.bitbucket.mycompany.com\/foobar/,
+      );
+    });
+  });
+
+  describe('handles repositories', () => {
+    it('output all repositories', async () => {
+      const target =
+        'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml';
+
+      client.listProjects.mockResolvedValue(
+        pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
+      );
+      client.listRepositories.mockResolvedValueOnce(
+        pagedResponse([
+          {
+            slug: 'backstage',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
+                },
+              ],
+            },
+          },
+        ]),
+      );
+      client.listRepositories.mockResolvedValueOnce(
+        pagedResponse([
+          {
+            slug: 'demo',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
+                },
+              ],
+            },
+          },
+        ]),
+      );
+
+      const actual = await readBitbucketOrg(client, target);
+      expect(actual.scanned).toBe(2);
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
+      });
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
+      });
+    });
+
+    it('output repositories with wildcards', async () => {
+      const target =
+        'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml';
+
+      client.listProjects.mockResolvedValue(
+        pagedResponse([{ key: 'backstage' }]),
+      );
+      client.listRepositories.mockResolvedValueOnce(
+        pagedResponse([
+          { slug: 'backstage' },
+          {
+            slug: 'techdocs-cli',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
+                },
+              ],
+            },
+          },
+          {
+            slug: 'techdocs-container',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
+                },
+              ],
+            },
+          },
+        ]),
+      );
+
+      const actual = await readBitbucketOrg(client, target);
+      expect(actual.scanned).toBe(3);
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
+      });
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
+      });
+    });
+    it('filter unrelated repositories', async () => {
+      const target =
+        'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml';
+
+      client.listProjects.mockResolvedValue(
+        pagedResponse([{ key: 'backstage' }]),
+      );
+      client.listRepositories.mockResolvedValue(
+        pagedResponse([
+          { slug: 'abstest' },
+          { slug: 'testxyz' },
+          {
+            slug: 'test',
+            links: {
+              self: [
+                {
+                  href:
+                    'https://bitbucket.mycompany.com/projects/backstage/repos/test',
+                },
+              ],
+            },
+          },
+        ]),
+      );
+
+      const actual = await readBitbucketOrg(client, target);
+      expect(actual.scanned).toBe(3);
+      expect(actual.matches).toContainEqual({
+        type: 'url',
+        target:
+          'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
+      });
+    });
+  });
+});
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Logger } from 'winston';
+import { Config } from '@backstage/config';
+
+import {
+  ScmIntegrationRegistry,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { LocationSpec } from '@backstage/catalog-model';
+import { BitbucketClient, paginated } from './bitbucket';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+import { results } from './index';
+
+export class BitbucketDiscoveryProcessor implements CatalogProcessor {
+  private readonly integrations: ScmIntegrationRegistry;
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const integrations = ScmIntegrations.fromConfig(config);
+
+    return new BitbucketDiscoveryProcessor({
+      ...options,
+      integrations,
+    });
+  }
+
+  constructor(options: {
+    integrations: ScmIntegrationRegistry;
+    logger: Logger;
+  }) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'bitbucket-discovery') {
+      return false;
+    }
+
+    const bitbucketConfig = this.integrations.bitbucket.byUrl(location.target)
+      ?.config;
+    if (!bitbucketConfig) {
+      throw new Error(
+        `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`,
+      );
+    } else if (bitbucketConfig.host === 'bitbucket.org') {
+      throw new Error(
+        `Component discovery for Bitbucket Cloud is not yet supported`,
+      );
+    }
+
+    const client = new BitbucketClient({
+      config: bitbucketConfig,
+    });
+    const startTimestamp = Date.now();
+    this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
+
+    const result = await readBitbucketOrg(client, location.target);
+
+    for (const repository of result.matches) {
+      emit(
+        results.location(
+          repository,
+          // Not all locations may actually exist, since the user defined them as a wildcard pattern.
+          // Thus, we emit them as optional and let the downstream processor find them while not outputting
+          // an error if it couldn't.
+          true,
+        ),
+      );
+    }
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`,
+    );
+
+    return true;
+  }
+}
+
+export async function readBitbucketOrg(
+  client: BitbucketClient,
+  target: string,
+): Promise<Result> {
+  const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target);
+  const projects = paginated(options => client.listProjects(options));
+  const result: Result = {
+    scanned: 0,
+    matches: [],
+  };
+
+  for await (const project of projects) {
+    if (!projectSearchPath.test(project.key)) {
+      continue;
+    }
+    const repositories = paginated(options =>
+      client.listRepositories(project.key, options),
+    );
+    for await (const repository of repositories) {
+      result.scanned++;
+
+      if (repoSearchPath.test(repository.slug)) {
+        result.matches.push({
+          type: 'url',
+          target: `${repository.links.self[0].href}${catalogPath}`,
+        });
+      }
+    }
+  }
+  return result;
+}
+
+function parseUrl(
+  urlString: string,
+): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } {
+  const url = new URL(urlString);
+  const path = url.pathname.substr(1).split('/');
+
+  // /projects/backstage/repos/techdocs-*/catalog-info.yaml
+  if (path.length > 3 && path[1].length && path[3].length) {
+    return {
+      projectSearchPath: escapeRegExp(decodeURIComponent(path[1])),
+      repoSearchPath: escapeRegExp(decodeURIComponent(path[3])),
+      catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`,
+    };
+  }
+
+  throw new Error(`Failed to parse ${urlString}`);
+}
+
+function escapeRegExp(str: string): RegExp {
+  return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
+}
+
+type Result = {
+  scanned: number;
+  matches: LocationSpec[];
+};
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fetch from 'cross-fetch';
+
+import {
+  BitbucketIntegrationConfig,
+  getBitbucketRequestOptions,
+} from '@backstage/integration';
+
+export class BitbucketClient {
+  private readonly config: BitbucketIntegrationConfig;
+
+  constructor(options: { config: BitbucketIntegrationConfig }) {
+    this.config = options.config;
+  }
+
+  async listProjects(options?: ListOptions): Promise<PagedResponse<any>> {
+    return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options);
+  }
+
+  async listRepositories(
+    projectKey: string,
+    options?: ListOptions,
+  ): Promise<PagedResponse<any>> {
+    return this.pagedRequest(
+      `${this.config.apiBaseUrl}/projects/${projectKey}/repos`,
+      options,
+    );
+  }
+
+  private async pagedRequest(
+    endpoint: string,
+    options?: ListOptions,
+  ): Promise<PagedResponse<any>> {
+    const request = new URL(endpoint);
+    for (const key in options) {
+      if (options[key]) {
+        request.searchParams.append(key, options[key]!.toString());
+      }
+    }
+
+    const response = await fetch(
+      request.toString(),
+      getBitbucketRequestOptions(this.config),
+    );
+    if (!response.ok) {
+      throw new Error(
+        `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
+          response.status
+        } - ${response.statusText}`,
+      );
+    }
+    return response.json().then(repositories => {
+      return repositories as PagedResponse<any>;
+    });
+  }
+}
+
+export type ListOptions = {
+  [key: string]: number | undefined;
+  limit?: number | undefined;
+  start?: number | undefined;
+};
+
+export type PagedResponse<T> = {
+  size: number;
+  limit: number;
+  start: number;
+  isLastPage: boolean;
+  values: T[];
+  nextPageStart: number;
+};
+
+export async function* paginated(
+  request: (options: ListOptions) => Promise<PagedResponse<any>>,
+  options?: ListOptions,
+) {
+  const opts = options || { start: 0 };
+  let res;
+  do {
+    res = await request(opts);
+    opts.start = res.nextPageStart;
+    for (const item of res.values) {
+      yield item;
+    }
+  } while (!res.isLastPage);
+}
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { BitbucketClient, paginated } from './client';
+export type { PagedResponse } from './client';
@@ -19,6 +19,7 @@ import * as results from './results';
 export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
 export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor';
 export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
+export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';
 export { FileReaderProcessor } from './FileReaderProcessor';
@@ -38,6 +38,7 @@ import {
 import { DatabaseManager } from '../database';
 import {
  AnnotateLocationEntityProcessor,
+  BitbucketDiscoveryProcessor,
  BuiltinKindsEntityProcessor,
  CatalogProcessor,
  CatalogProcessorParser,
@@ -304,6 +305,7 @@ export class CatalogBuilder {
    if (!this.processorsReplace) {
      processors.push(
        new FileReaderProcessor(),
+        BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
        GithubDiscoveryProcessor.fromConfig(config, { logger }),
        GithubOrgReaderProcessor.fromConfig(config, { logger }),
        LdapOrgReaderProcessor.fromConfig(config, { logger }),