fix(catalog-backend): support Bitbucket servers with custom BaseURLs

Previously, the `BitbucketDiscoveryProcessor` assumed that all Bitbucket servers would have paths formed in the pattern of `/projects/{project}/repos/{repo}`.  However, since Bitbucket servers have the ability to configure custom baseUrls, this assumption does not always hold true. Instead of evaluating project and repo terms from the full target path, the processor will find the `/projects/` url segment, and start path evaluations from its index instead. This path substring is then used to determine the `projectSearchPath` and the `repoSearchPath`. Support is also added to accept search params for the `catalogPath` as well.

Fixes #8771

Signed-off-by: Dhruval Darji <dhruvaldarji@gmail.com>

.changeset/metal-worms-flash.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Provide support for Bitbucket servers with custom BaseURLs.

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts

@@ -29,7 +29,10 @@ import { setupServer } from 'msw/node';
 
 const server = setupServer();
 
-function setupStubs(projects: any[]) {
+function setupStubs(
+  projects: any[],
+  bitbucketBaseUrl = `https://bitbucket.mycompany.com`,
+) {
   function pagedResponse(values: any): PagedResponse<any> {
     return {
       values: values,
@@ -42,7 +45,7 @@ function setupStubs(projects: any[]) {
     repos: string[],
   ): RequestHandler<any, any> {
     return rest.get(
-      `https://bitbucket.mycompany.com/api/rest/1.0/projects/${project}/repos`,
+      `${bitbucketBaseUrl}/api/rest/1.0/projects/${project}/repos`,
       (_, res, ctx) => {
         const response = [];
         for (const repo of repos) {
@@ -51,7 +54,7 @@ function setupStubs(projects: any[]) {
             links: {
               self: [
                 {
-                  href: `https://bitbucket.mycompany.com/projects/${project}/repos/${repo}/browse`,
+                  href: `${bitbucketBaseUrl}/projects/${project}/repos/${repo}/browse`,
                 },
               ],
             },
@@ -63,20 +66,17 @@ function setupStubs(projects: any[]) {
   }
 
   server.use(
-    rest.get(
-      `https://bitbucket.mycompany.com/api/rest/1.0/projects`,
-      (_, res, ctx) => {
-        return res(
-          ctx.json(
-            pagedResponse(
-              projects.map(p => {
-                return { key: p.key };
-              }),
-            ),
+    rest.get(`${bitbucketBaseUrl}/api/rest/1.0/projects`, (_, res, ctx) => {
+      return res(
+        ctx.json(
+          pagedResponse(
+            projects.map(p => {
+              return { key: p.key };
+            }),
           ),
-        );
-      },
-    ),
+        ),
+      );
+    }),
   );
 
   for (const project of projects) {
@@ -249,6 +249,30 @@ describe('BitbucketDiscoveryProcessor', () => {
         optional: true,
       });
     });
+
+    it('output repositories by target search ref', async () => {
+      setupStubs([{ key: 'demo', repos: ['demo'] }]);
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.mycompany.com/projects/demo/repos/demo/catalog.yaml?ref=branch-name',
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml?ref=branch-name',
+          presence: 'optional',
+        },
+        optional: true,
+      });
+    });
+
     it('filter unrelated repositories', async () => {
       setupStubs([{ key: 'backstage', repos: ['test', 'abctest', 'testxyz'] }]);
       const location: LocationSpec = {
@@ -302,6 +326,177 @@ describe('BitbucketDiscoveryProcessor', () => {
     });
   });
 
+  describe('handles organisation repositories with a custom baseURL', () => {
+    const processor = BitbucketDiscoveryProcessor.fromConfig(
+      new ConfigReader({
+        integrations: {
+          bitbucket: [
+            {
+              host: 'bitbucket.mycompany.com',
+              token: 'blob',
+              apiBaseUrl:
+                'https://bitbucket.mycompany.com/custom-path/api/rest/1.0',
+            },
+          ],
+        },
+      }),
+      { logger: getVoidLogger() },
+    );
+
+    it('output all repositories', async () => {
+      setupStubs(
+        [
+          { key: 'backstage', repos: ['backstage'] },
+          { key: 'demo', repos: ['demo'] },
+        ],
+        'https://bitbucket.mycompany.com/custom-path',
+      );
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.mycompany.com/custom-path/projects/*/repos/*/catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/backstage/browse/catalog.yaml',
+          presence: 'optional',
+        },
+        optional: true,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/custom-path/projects/demo/repos/demo/browse/catalog.yaml',
+          presence: 'optional',
+        },
+        optional: true,
+      });
+    });
+
+    it('output repositories with wildcards', async () => {
+      setupStubs(
+        [
+          { key: 'backstage', repos: ['backstage', 'techdocs-cli'] },
+          { key: 'demo', repos: ['demo'] },
+        ],
+        'https://bitbucket.mycompany.com/custom-path',
+      );
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/techdocs-*/catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
+          presence: 'optional',
+        },
+        optional: true,
+      });
+    });
+
+    it('output repositories by target search ref', async () => {
+      setupStubs(
+        [{ key: 'demo', repos: ['demo'] }],
+        'https://bitbucket.mycompany.com/custom-path',
+      );
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.mycompany.com/custom-path/projects/demo/repos/demo/catalog.yaml?ref=branch-name',
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/custom-path/projects/demo/repos/demo/browse/catalog.yaml?ref=branch-name',
+          presence: 'optional',
+        },
+        optional: true,
+      });
+    });
+
+    it('filter unrelated repositories', async () => {
+      setupStubs(
+        [{ key: 'backstage', repos: ['test', 'abctest', 'testxyz'] }],
+        'https://bitbucket.mycompany.com/custom-path',
+      );
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/test/catalog.yaml',
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/test/browse/catalog.yaml',
+          presence: 'optional',
+        },
+        optional: true,
+      });
+    });
+
+    it.each`
+      target
+      ${'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/*'}
+      ${'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/*/'}
+      ${'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/techdocs-*/'}
+    `("target '$target' adds default path to catalog", async ({ target }) => {
+      setupStubs(
+        [{ key: 'backstage', repos: ['techdocs-cli'] }],
+        'https://bitbucket.mycompany.com/custom-path',
+      );
+
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target: target,
+      };
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/techdocs-cli/browse/catalog-info.yaml',
+          presence: 'optional',
+        },
+        optional: true,
+      });
+    });
+  });
+
   describe('handles cloud repositories', () => {
     const processor = BitbucketDiscoveryProcessor.fromConfig(
       new ConfigReader({

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts

@@ -230,14 +230,18 @@ function parseUrl(urlString: string): {
   catalogPath: string;
 } {
   const url = new URL(urlString);
-  const path = url.pathname.substr(1).split('/');
+  const indexOfProjectSegment =
+    url.pathname.toLowerCase().indexOf('/projects/') + 1;
+  const path = url.pathname.substr(indexOfProjectSegment).split('/');
 
   // /projects/backstage/repos/techdocs-*/catalog-info.yaml
   if (path.length > 3 && path[1].length && path[3].length) {
     return {
       projectSearchPath: escapeRegExp(decodeURIComponent(path[1])),
       repoSearchPath: escapeRegExp(decodeURIComponent(path[3])),
-      catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`,
+      catalogPath: `/${decodeURIComponent(
+        path.slice(4).join('/') + url.search,
+      )}`,
     };
   }