From 407aefba221b4974d5ab37cd68b844b273230587 Mon Sep 17 00:00:00 2001
From: Bart Breen <bart.breen@twobulls.com>
Date: Tue, 27 Jul 2021 15:17:30 +1000
Subject: [PATCH] Fixed paging and authenticated requests

Signed-off-by: Bart Breen <bart.breen@twobulls.com>
---
 .../processors/BitbucketDiscoveryProcessor.test.ts | 14 +++++++-------
 .../processors/BitbucketDiscoveryProcessor.ts      |  2 +-
 .../src/ingestion/processors/bitbucket/client.ts   |  3 ++-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts
index 43efa8e5d6..559174054f 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts
@@ -106,8 +106,8 @@ function setupBitbucketCloudStubs(
               repositories.map(r => ({
                 ...r,
                 links: {
-                  source: {
-                    href: `https://api.bitbucket.org/2.0/repositories/${workspace}/${r.slug}/src`,
+                  html: {
+                    href: `https://bitbucket.org/${workspace}/${r.slug}`,
                   },
                 },
               })),
@@ -332,7 +332,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml',
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
         },
         optional: true,
       });
@@ -341,7 +341,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-two/src/master/catalog.yaml',
+            'https://bitbucket.org/myworkspace/repository-two/src/master/catalog.yaml',
         },
         optional: true,
       });
@@ -367,7 +367,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml',
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
         },
         optional: true,
       });
@@ -393,7 +393,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-three/src/master/catalog.yaml',
+            'https://bitbucket.org/myworkspace/repository-three/src/master/catalog.yaml',
         },
         optional: true,
       });
@@ -424,7 +424,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog-info.yaml',
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
         },
         optional: true,
       });
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts
index dd5ff0d4af..3521c98af5 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts
@@ -124,7 +124,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
       const mainbranch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
       for await (const entity of this.parser({
         integration,
-        target: `${repository.links.source.href}/${mainbranch}${catalogPath}`,
+        target: `${repository.links.html.href}/src/${mainbranch}${catalogPath}`,
         logger: this.logger,
       })) {
         emit(entity);
diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts
index bdea349180..485e248a76 100644
--- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts
@@ -125,6 +125,7 @@ export type PagedResponse<T> = {
 export type ListOptions20 = {
   [key: string]: number | undefined;
   page?: number | undefined;
+  pagelen?: number | undefined;
 };
 
 export type PagedResponse20<T> = {
@@ -154,7 +155,7 @@ export async function* paginated20<T = any>(
   request: (options: ListOptions20) => Promise<PagedResponse20<T>>,
   options?: ListOptions20,
 ) {
-  const opts = options || { page: 1 };
+  const opts = options || { page: 1, pagelen: 100 };
   let res;
   do {
     res = await request(opts);