From 2aded73e384cb97bc7ed4a663ef7e94571f4d1aa Mon Sep 17 00:00:00 2001 From: Joseph Roberto Date: Thu, 18 Sep 2025 12:45:13 -0500 Subject: [PATCH] feat: Allow configurable pagelength for Bitbucket Cloud provider Signed-off-by: Joseph Roberto --- .changeset/tame-hairs-smash.md | 6 ++++ plugins/bitbucket-cloud-common/report.api.md | 2 ++ .../src/BitbucketCloudClient.test.ts | 35 +++++++++++++++++++ .../src/BitbucketCloudClient.ts | 2 ++ .../src/pagination.test.ts | 16 ++++++++- .../bitbucket-cloud-common/src/pagination.ts | 7 ++-- .../config.d.ts | 10 ++++++ .../providers/BitbucketCloudEntityProvider.ts | 2 +- ...BitbucketCloudEntityProviderConfig.test.ts | 22 ++++++++++++ .../BitbucketCloudEntityProviderConfig.ts | 4 +++ 10 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 .changeset/tame-hairs-smash.md diff --git a/.changeset/tame-hairs-smash.md b/.changeset/tame-hairs-smash.md new file mode 100644 index 0000000000..d74b2ffeb7 --- /dev/null +++ b/.changeset/tame-hairs-smash.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch +'@backstage/plugin-bitbucket-cloud-common': patch +--- + +Allow for passing a pagelen parameter to configure the pagelength property of the BitbucketCloudEntityProvider searchCode pagination to resolve [bug](https://jira.atlassian.com/browse/BCLOUD-23644) pertaining to duplicate results being returned. diff --git a/plugins/bitbucket-cloud-common/report.api.md b/plugins/bitbucket-cloud-common/report.api.md index c53863b92f..11f0c93593 100644 --- a/plugins/bitbucket-cloud-common/report.api.md +++ b/plugins/bitbucket-cloud-common/report.api.md @@ -36,6 +36,7 @@ export class BitbucketCloudClient { workspace: string, query: string, options?: FilterAndSortOptions & PartialResponseOptions, + pagelen?: number, ): WithPagination; } @@ -517,6 +518,7 @@ export class WithPagination< constructor( createUrl: (options: PaginationOptions) => URL, fetch: (url: URL) => Promise, + pagelen?: number | undefined, ); // (undocumented) getPage(options?: PaginationOptions): Promise; diff --git a/plugins/bitbucket-cloud-common/src/BitbucketCloudClient.test.ts b/plugins/bitbucket-cloud-common/src/BitbucketCloudClient.test.ts index 7558c92138..ed696ec6fb 100644 --- a/plugins/bitbucket-cloud-common/src/BitbucketCloudClient.test.ts +++ b/plugins/bitbucket-cloud-common/src/BitbucketCloudClient.test.ts @@ -79,6 +79,41 @@ describe('BitbucketCloudClient', () => { expect(results[0].file!.path).toEqual('path/to/file'); }); + it('searchCode with custom pagelen', async () => { + server.use( + rest.get( + `https://api.bitbucket.org/2.0/workspaces/ws/search/code`, + (req, res, ctx) => { + const pagelen = req.url.searchParams.get('pagelen'); + expect(pagelen).toBe('50'); + + const response: Models.SearchResultPage = { + values: [ + { + content_match_count: 1, + file: { + type: 'commit_file', + path: 'path/to/file', + }, + }, + ], + }; + return res(ctx.json(response)); + }, + ), + ); + + const pagination = client.searchCode('ws', 'query', undefined, 50); + + const results = []; + for await (const result of pagination.iterateResults()) { + results.push(result); + } + + expect(results).toHaveLength(1); + expect(results[0].file!.path).toEqual('path/to/file'); + }); + it('listRepositoriesByWorkspace', async () => { server.use( rest.get( diff --git a/plugins/bitbucket-cloud-common/src/BitbucketCloudClient.ts b/plugins/bitbucket-cloud-common/src/BitbucketCloudClient.ts index 86f56dd70d..d1d4a35561 100644 --- a/plugins/bitbucket-cloud-common/src/BitbucketCloudClient.ts +++ b/plugins/bitbucket-cloud-common/src/BitbucketCloudClient.ts @@ -40,6 +40,7 @@ export class BitbucketCloudClient { workspace: string, query: string, options?: FilterAndSortOptions & PartialResponseOptions, + pagelen?: number, ): WithPagination { const workspaceEnc = encodeURIComponent(workspace); return new WithPagination( @@ -50,6 +51,7 @@ export class BitbucketCloudClient { search_query: query, }), url => this.getTypeMapped(url), + pagelen, ); } diff --git a/plugins/bitbucket-cloud-common/src/pagination.test.ts b/plugins/bitbucket-cloud-common/src/pagination.test.ts index f263f51897..c355a798e4 100644 --- a/plugins/bitbucket-cloud-common/src/pagination.test.ts +++ b/plugins/bitbucket-cloud-common/src/pagination.test.ts @@ -24,7 +24,7 @@ interface TestResultItem { interface TestPage extends Models.Paginated {} describe('WithPagination', () => { - const createPagination = () => + const createPagination = (pagelen?: number) => new WithPagination( opts => new URL( @@ -45,6 +45,7 @@ describe('WithPagination', () => { ], }; }, + pagelen, ); it('iterateResults', async () => { @@ -121,4 +122,17 @@ describe('WithPagination', () => { ); }); }); + + it('uses custom pagelen when provided', async () => { + const pagination = createPagination(50); + + const page = await pagination.getPage(); + + expect(page.page).toEqual(1); + expect(page.next).toEqual('http://localhost/create-url?page=2&pagelen=50'); + expect(page.values).toHaveLength(1); + expect((page.values! as TestResultItem[])[0].url).toEqual( + 'http://localhost/create-url?page=1&pagelen=50', + ); + }); }); diff --git a/plugins/bitbucket-cloud-common/src/pagination.ts b/plugins/bitbucket-cloud-common/src/pagination.ts index 98e4f9f9d7..5d699a4a6f 100644 --- a/plugins/bitbucket-cloud-common/src/pagination.ts +++ b/plugins/bitbucket-cloud-common/src/pagination.ts @@ -30,10 +30,11 @@ export class WithPagination< constructor( private readonly createUrl: (options: PaginationOptions) => URL, private readonly fetch: (url: URL) => Promise, + private readonly pagelen?: number, ) {} getPage(options?: PaginationOptions): Promise { - const opts = { page: 1, pagelen: 100, ...options }; + const opts = { page: 1, pagelen: this.pagelen ?? 100, ...options }; const url = this.createUrl(opts); return this.fetch(url); } @@ -41,7 +42,7 @@ export class WithPagination< async *iteratePages( options?: PaginationOptions, ): AsyncGenerator { - const opts = { page: 1, pagelen: 100, ...options }; + const opts = { page: 1, pagelen: this.pagelen ?? 100, ...options }; let url: URL | undefined = this.createUrl(opts); let res; do { @@ -52,7 +53,7 @@ export class WithPagination< } async *iterateResults(options?: PaginationOptions) { - const opts = { page: 1, pagelen: 100, ...options }; + const opts = { page: 1, pagelen: this.pagelen ?? 100, ...options }; let url: URL | undefined = this.createUrl(opts); let res; do { diff --git a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts index 1051976b24..bc6d618aa8 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts @@ -59,6 +59,11 @@ export interface Config { * (Optional) TaskScheduleDefinition for the discovery. */ schedule?: SchedulerServiceTaskScheduleDefinitionConfig; + /** + * (Optional) Number of results to fetch per page from Bitbucket API. Default to 100. + * @visibility frontend + */ + pagelen?: number; } | { [name: string]: { @@ -92,6 +97,11 @@ export interface Config { * (Optional) TaskScheduleDefinition for the discovery. */ schedule?: SchedulerServiceTaskScheduleDefinitionConfig; + /** + * (Optional) Number of results to fetch per page from Bitbucket API. Default to 100. + * @visibility frontend + */ + pagelen?: number; }; }; }; diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts index 64a58c2924..33327e1f33 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts @@ -409,7 +409,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { ].join(','); const searchResults = this.client - .searchCode(workspace, query, { fields }) + .searchCode(workspace, query, { fields }, this.config.pagelen) .iterateResults(); const result: IngestionTarget[] = []; diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts index 7f13616586..e039708099 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts @@ -92,6 +92,7 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: undefined, }, + pagelen: undefined, }); expect(providerConfigs[1]).toEqual({ id: 'providerCustomCatalogPath', @@ -101,6 +102,7 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: undefined, }, + pagelen: undefined, }); expect(providerConfigs[2]).toEqual({ id: 'providerWithProjectKeyFilter', @@ -110,6 +112,7 @@ describe('readProviderConfigs', () => { projectKey: /^projectKey.*filter$/, repoSlug: undefined, }, + pagelen: undefined, }); expect(providerConfigs[3]).toEqual({ id: 'providerWithRepoSlugFilter', @@ -119,6 +122,7 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: /^repoSlug.*filter$/, }, + pagelen: undefined, }); expect(providerConfigs[4]).toEqual({ id: 'providerWithSchedule', @@ -134,6 +138,24 @@ describe('readProviderConfigs', () => { minutes: 3, }, }, + pagelen: undefined, }); }); + + it('provider config with pagelen', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + workspace: 'test-ws', + pagelen: 50, + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].pagelen).toEqual(50); + }); }); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts index 5a806d2e0b..0a8259e4d1 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts @@ -32,6 +32,7 @@ export type BitbucketCloudEntityProviderConfig = { repoSlug?: RegExp; }; schedule?: SchedulerServiceTaskScheduleDefinition; + pagelen?: number; }; export function readProviderConfigs( @@ -72,6 +73,8 @@ function readProviderConfig( ) : undefined; + const pagelen = config.getOptionalNumber('pagelen'); + return { id, catalogPath, @@ -83,6 +86,7 @@ function readProviderConfig( repoSlug: repoSlugPattern ? compileRegExp(repoSlugPattern) : undefined, }, schedule, + pagelen, }; }