feat: Allow configurable pagelength for Bitbucket Cloud provider

Signed-off-by: Joseph Roberto <robertoj921@yahoo.com>

.changeset/tame-hairs-smash.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch
+'@backstage/plugin-bitbucket-cloud-common': patch
+---
+
+Allow for passing a pagelen parameter to configure the pagelength property of the BitbucketCloudEntityProvider searchCode pagination to resolve [bug](https://jira.atlassian.com/browse/BCLOUD-23644) pertaining to duplicate results being returned.

plugins/bitbucket-cloud-common/report.api.md

@@ -36,6 +36,7 @@ export class BitbucketCloudClient {
     workspace: string,
     query: string,
     options?: FilterAndSortOptions & PartialResponseOptions,
+    pagelen?: number,
   ): WithPagination<Models.SearchResultPage, Models.SearchCodeSearchResult>;
 }
 
@@ -517,6 +518,7 @@ export class WithPagination<
   constructor(
     createUrl: (options: PaginationOptions) => URL,
     fetch: (url: URL) => Promise<TPage>,
+    pagelen?: number | undefined,
   );
   // (undocumented)
   getPage(options?: PaginationOptions): Promise<TPage>;

plugins/bitbucket-cloud-common/src/BitbucketCloudClient.test.ts

@@ -79,6 +79,41 @@ describe('BitbucketCloudClient', () => {
     expect(results[0].file!.path).toEqual('path/to/file');
   });
 
+  it('searchCode with custom pagelen', async () => {
+    server.use(
+      rest.get(
+        `https://api.bitbucket.org/2.0/workspaces/ws/search/code`,
+        (req, res, ctx) => {
+          const pagelen = req.url.searchParams.get('pagelen');
+          expect(pagelen).toBe('50');
+
+          const response: Models.SearchResultPage = {
+            values: [
+              {
+                content_match_count: 1,
+                file: {
+                  type: 'commit_file',
+                  path: 'path/to/file',
+                },
+              },
+            ],
+          };
+          return res(ctx.json(response));
+        },
+      ),
+    );
+
+    const pagination = client.searchCode('ws', 'query', undefined, 50);
+
+    const results = [];
+    for await (const result of pagination.iterateResults()) {
+      results.push(result);
+    }
+
+    expect(results).toHaveLength(1);
+    expect(results[0].file!.path).toEqual('path/to/file');
+  });
+
   it('listRepositoriesByWorkspace', async () => {
     server.use(
       rest.get(

plugins/bitbucket-cloud-common/src/BitbucketCloudClient.ts

@@ -40,6 +40,7 @@ export class BitbucketCloudClient {
     workspace: string,
     query: string,
     options?: FilterAndSortOptions & PartialResponseOptions,
+    pagelen?: number,
   ): WithPagination<Models.SearchResultPage, Models.SearchCodeSearchResult> {
     const workspaceEnc = encodeURIComponent(workspace);
     return new WithPagination(
@@ -50,6 +51,7 @@ export class BitbucketCloudClient {
           search_query: query,
         }),
       url => this.getTypeMapped(url),
+      pagelen,
     );
   }
 

plugins/bitbucket-cloud-common/src/pagination.test.ts

@@ -24,7 +24,7 @@ interface TestResultItem {
 interface TestPage extends Models.Paginated<TestResultItem> {}
 
 describe('WithPagination', () => {
-  const createPagination = () =>
+  const createPagination = (pagelen?: number) =>
     new WithPagination<TestPage, TestResultItem>(
       opts =>
         new URL(
@@ -45,6 +45,7 @@ describe('WithPagination', () => {
           ],
         };
       },
+      pagelen,
     );
 
   it('iterateResults', async () => {
@@ -121,4 +122,17 @@ describe('WithPagination', () => {
       );
     });
   });
+
+  it('uses custom pagelen when provided', async () => {
+    const pagination = createPagination(50);
+
+    const page = await pagination.getPage();
+
+    expect(page.page).toEqual(1);
+    expect(page.next).toEqual('http://localhost/create-url?page=2&pagelen=50');
+    expect(page.values).toHaveLength(1);
+    expect((page.values! as TestResultItem[])[0].url).toEqual(
+      'http://localhost/create-url?page=1&pagelen=50',
+    );
+  });
 });

plugins/bitbucket-cloud-common/src/pagination.ts

@@ -30,10 +30,11 @@ export class WithPagination<
   constructor(
     private readonly createUrl: (options: PaginationOptions) => URL,
     private readonly fetch: (url: URL) => Promise<TPage>,
+    private readonly pagelen?: number,
   ) {}
 
   getPage(options?: PaginationOptions): Promise<TPage> {
-    const opts = { page: 1, pagelen: 100, ...options };
+    const opts = { page: 1, pagelen: this.pagelen ?? 100, ...options };
     const url = this.createUrl(opts);
     return this.fetch(url);
   }
@@ -41,7 +42,7 @@ export class WithPagination<
   async *iteratePages(
     options?: PaginationOptions,
   ): AsyncGenerator<TPage, void> {
-    const opts = { page: 1, pagelen: 100, ...options };
+    const opts = { page: 1, pagelen: this.pagelen ?? 100, ...options };
     let url: URL | undefined = this.createUrl(opts);
     let res;
     do {
@@ -52,7 +53,7 @@ export class WithPagination<
   }
 
   async *iterateResults(options?: PaginationOptions) {
-    const opts = { page: 1, pagelen: 100, ...options };
+    const opts = { page: 1, pagelen: this.pagelen ?? 100, ...options };
     let url: URL | undefined = this.createUrl(opts);
     let res;
     do {

plugins/catalog-backend-module-bitbucket-cloud/config.d.ts

@@ -59,6 +59,11 @@ export interface Config {
              * (Optional) TaskScheduleDefinition for the discovery.
              */
             schedule?: SchedulerServiceTaskScheduleDefinitionConfig;
+            /**
+             * (Optional) Number of results to fetch per page from Bitbucket API. Default to 100.
+             * @visibility frontend
+             */
+            pagelen?: number;
           }
         | {
             [name: string]: {
@@ -92,6 +97,11 @@ export interface Config {
                * (Optional) TaskScheduleDefinition for the discovery.
                */
               schedule?: SchedulerServiceTaskScheduleDefinitionConfig;
+              /**
+               * (Optional) Number of results to fetch per page from Bitbucket API. Default to 100.
+               * @visibility frontend
+               */
+              pagelen?: number;
             };
           };
     };

plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts

@@ -409,7 +409,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
     ].join(',');
 
     const searchResults = this.client
-      .searchCode(workspace, query, { fields })
+      .searchCode(workspace, query, { fields }, this.config.pagelen)
       .iterateResults();
 
     const result: IngestionTarget[] = [];

plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts

@@ -92,6 +92,7 @@ describe('readProviderConfigs', () => {
         projectKey: undefined,
         repoSlug: undefined,
       },
+      pagelen: undefined,
     });
     expect(providerConfigs[1]).toEqual({
       id: 'providerCustomCatalogPath',
@@ -101,6 +102,7 @@ describe('readProviderConfigs', () => {
         projectKey: undefined,
         repoSlug: undefined,
       },
+      pagelen: undefined,
     });
     expect(providerConfigs[2]).toEqual({
       id: 'providerWithProjectKeyFilter',
@@ -110,6 +112,7 @@ describe('readProviderConfigs', () => {
         projectKey: /^projectKey.*filter$/,
         repoSlug: undefined,
       },
+      pagelen: undefined,
     });
     expect(providerConfigs[3]).toEqual({
       id: 'providerWithRepoSlugFilter',
@@ -119,6 +122,7 @@ describe('readProviderConfigs', () => {
         projectKey: undefined,
         repoSlug: /^repoSlug.*filter$/,
       },
+      pagelen: undefined,
     });
     expect(providerConfigs[4]).toEqual({
       id: 'providerWithSchedule',
@@ -134,6 +138,24 @@ describe('readProviderConfigs', () => {
           minutes: 3,
         },
       },
+      pagelen: undefined,
     });
   });
+
+  it('provider config with pagelen', () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          bitbucketCloud: {
+            workspace: 'test-ws',
+            pagelen: 50,
+          },
+        },
+      },
+    });
+    const providerConfigs = readProviderConfigs(config);
+
+    expect(providerConfigs).toHaveLength(1);
+    expect(providerConfigs[0].pagelen).toEqual(50);
+  });
 });

plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts

@@ -32,6 +32,7 @@ export type BitbucketCloudEntityProviderConfig = {
     repoSlug?: RegExp;
   };
   schedule?: SchedulerServiceTaskScheduleDefinition;
+  pagelen?: number;
 };
 
 export function readProviderConfigs(
@@ -72,6 +73,8 @@ function readProviderConfig(
       )
     : undefined;
 
+  const pagelen = config.getOptionalNumber('pagelen');
+
   return {
     id,
     catalogPath,
@@ -83,6 +86,7 @@ function readProviderConfig(
       repoSlug: repoSlugPattern ? compileRegExp(repoSlugPattern) : undefined,
     },
     schedule,
+    pagelen,
   };
 }