Fixed paging and authenticated requests

Signed-off-by: Bart Breen <bart.breen@twobulls.com>

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts

@@ -106,8 +106,8 @@ function setupBitbucketCloudStubs(
               repositories.map(r => ({
                 ...r,
                 links: {
-                  source: {
-                    href: `https://api.bitbucket.org/2.0/repositories/${workspace}/${r.slug}/src`,
+                  html: {
+                    href: `https://bitbucket.org/${workspace}/${r.slug}`,
                   },
                 },
               })),
@@ -332,7 +332,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml',
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
         },
         optional: true,
       });
@@ -341,7 +341,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-two/src/master/catalog.yaml',
+            'https://bitbucket.org/myworkspace/repository-two/src/master/catalog.yaml',
         },
         optional: true,
       });
@@ -367,7 +367,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml',
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
         },
         optional: true,
       });
@@ -393,7 +393,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-three/src/master/catalog.yaml',
+            'https://bitbucket.org/myworkspace/repository-three/src/master/catalog.yaml',
         },
         optional: true,
       });
@@ -424,7 +424,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog-info.yaml',
+            'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
         },
         optional: true,
       });

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts

@@ -124,7 +124,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
       const mainbranch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
       for await (const entity of this.parser({
         integration,
-        target: `${repository.links.source.href}/${mainbranch}${catalogPath}`,
+        target: `${repository.links.html.href}/src/${mainbranch}${catalogPath}`,
         logger: this.logger,
       })) {
         emit(entity);

plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts

@@ -125,6 +125,7 @@ export type PagedResponse<T> = {
 export type ListOptions20 = {
   [key: string]: number | undefined;
   page?: number | undefined;
+  pagelen?: number | undefined;
 };
 
 export type PagedResponse20<T> = {
@@ -154,7 +155,7 @@ export async function* paginated20<T = any>(
   request: (options: ListOptions20) => Promise<PagedResponse20<T>>,
   options?: ListOptions20,
 ) {
-  const opts = options || { page: 1 };
+  const opts = options || { page: 1, pagelen: 100 };
   let res;
   do {
     res = await request(opts);