From 3574c51b248cceab6c75758a9e20ed926f6ac30f Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Thu, 26 Sep 2024 15:36:34 +0200 Subject: [PATCH 01/11] feat: add level config to catalog-backend-module-bitbucket-cloud plugin Signed-off-by: Benjamin Janssens --- .../config.d.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts index 1051976b24..6ba4cb135d 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts @@ -59,6 +59,14 @@ export interface Config { * (Optional) TaskScheduleDefinition for the discovery. */ schedule?: SchedulerServiceTaskScheduleDefinitionConfig; + /** + * (Optional) On what level discovery should take place, affecting Bitbucket Cloud API limits. + * + * Possible values: + * - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace. + * - `project`: 1 API call per project, limited to 900 repositories per project. + */ + level?: 'workspace' | 'project'; } | { [name: string]: { @@ -92,6 +100,14 @@ export interface Config { * (Optional) TaskScheduleDefinition for the discovery. */ schedule?: SchedulerServiceTaskScheduleDefinitionConfig; + /** + * (Optional) On what level discovery should take place, affecting Bitbucket Cloud API limits. + * + * Possible values: + * - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace. + * - `project`: 1 API call per project, limited to 900 repositories per project. + */ + level?: 'workspace' | 'project'; }; }; }; From 8dbc4ab48246580d2d0ceefa587b5deede7e8507 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Thu, 26 Sep 2024 16:03:22 +0200 Subject: [PATCH 02/11] chore: add level config to BitbucketCloudEntityProviderConfig Signed-off-by: Benjamin Janssens --- .../src/providers/BitbucketCloudEntityProviderConfig.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts index 5a806d2e0b..1326c53e29 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts @@ -32,6 +32,7 @@ export type BitbucketCloudEntityProviderConfig = { repoSlug?: RegExp; }; schedule?: SchedulerServiceTaskScheduleDefinition; + level?: 'workspace' | 'project'; }; export function readProviderConfigs( From a1219e94cb2eb103e8cbed663a17fe0f4f37e487 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Fri, 27 Sep 2024 11:11:12 +0200 Subject: [PATCH 03/11] feat: implement project-level Bitbucket Cloud discovery Signed-off-by: Benjamin Janssens --- .../providers/BitbucketCloudEntityProvider.ts | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts index 6b19bea4f2..6826804053 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts @@ -207,7 +207,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { logger.info('Discovering catalog files in Bitbucket Cloud repositories'); - const targets = await this.findCatalogFiles(); + const targets = await this.findCatalogFiles(this.config.level); const entities = this.toDeferredEntities(targets); await this.connection.applyMutation({ @@ -271,7 +271,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { // Hence, we will just trigger a refresh for catalog file(s) within the repository // if we get notified about changes there. - const targets = await this.findCatalogFiles(repoSlug); + const targets = await this.findCatalogFiles('workspace', repoSlug); const { token } = await this.tokenManager!.getToken(); const existing = await this.findExistingLocations(repoUrl, token); @@ -334,6 +334,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { } private async findCatalogFiles( + level: 'workspace' | 'project' = 'workspace', repoSlug?: string, ): Promise { const workspace = this.config.workspace; @@ -343,6 +344,32 @@ export class BitbucketCloudEntityProvider implements EntityProvider { catalogPath.lastIndexOf('/') + 1, ); + const optRepoFilter = repoSlug ? ` repo:${repoSlug}` : ''; + const query = `"${catalogFilename}" path:${catalogPath}${optRepoFilter}`; + + if (level === 'project') { + const projects = this.client + .listProjectsByWorkspace(workspace) + .iterateResults(); + + const results: IngestionTarget[] = []; + + for await (const project of projects) { + const projectQuery = `${query} project:${project.key}`; + const result = await this.processQuery(workspace, projectQuery); + results.push(...result); + } + + return results; + } + + return this.processQuery(workspace, query); + } + + private async processQuery( + workspace: string, + query: string, + ): Promise { // load all fields relevant for creating refs later, but not more const fields = [ // exclude code/content match details @@ -358,8 +385,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { // ...except the one we need '+values.file.commit.repository.links.html.href', ].join(','); - const optRepoFilter = repoSlug ? ` repo:${repoSlug}` : ''; - const query = `"${catalogFilename}" path:${catalogPath}${optRepoFilter}`; + const searchResults = this.client .searchCode(workspace, query, { fields }) .iterateResults(); From bcb31c766cf689e0d8806f5154dd7729b461f5a2 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Fri, 27 Sep 2024 12:09:00 +0200 Subject: [PATCH 04/11] test: add tests; fix config Signed-off-by: Benjamin Janssens --- .../BitbucketCloudEntityProvider.test.ts | 192 ++++++++++++++++++ .../providers/BitbucketCloudEntityProvider.ts | 2 +- .../BitbucketCloudEntityProviderConfig.ts | 9 +- 3 files changed, 201 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts index 262582b99a..f95cb34025 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts @@ -87,6 +87,23 @@ describe('BitbucketCloudEntityProvider', () => { }, }, }); + const projectLevelConfig = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + myProvider: { + workspace: 'test-ws', + catalogPath: 'catalog-custom.yaml', + filters: { + projectKey: 'test-.*', + repoSlug: 'test-.*', + }, + level: 'project', + }, + }, + }, + }, + }); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -419,6 +436,181 @@ describe('BitbucketCloudEntityProvider', () => { }); }); + it('apply full update on scheduled execution on project-level', async () => { + const provider = BitbucketCloudEntityProvider.fromConfig( + projectLevelConfig, + { + logger, + schedule, + }, + )[0]; + expect(provider.getProviderName()).toEqual( + 'bitbucketCloud-provider:myProvider', + ); + + server.use( + rest.get( + `https://api.bitbucket.org/2.0/workspaces/test-ws/projects`, + (_req, res, ctx) => { + const response = { + values: [ + { + key: 'TEST', + }, + ], + }; + return res(ctx.json(response)); + }, + ), + rest.get( + `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, + (req, res, ctx) => { + const query = req.url.searchParams.get('search_query'); + if (!query || !query.includes('project:TEST')) { + return res(ctx.json({ values: [] })); + } + + const response = { + values: [ + { + // skipped as empty + path_matches: [], + file: { + type: 'commit_file', + path: 'path/to/ignored/file', + }, + }, + { + path_matches: [ + { + match: true, + text: 'catalog-custom.yaml', + }, + ], + file: { + type: 'commit_file', + path: 'custom/path/catalog-custom.yaml', + commit: { + repository: { + // skipped as no match with filter + slug: 'repo', + project: { + key: 'test-project', + }, + mainbranch: { + name: 'main', + }, + links: { + html: { + href: 'https://bitbucket.org/test-ws/repo', + }, + }, + }, + }, + }, + }, + { + path_matches: [ + { + match: true, + text: 'catalog-custom.yaml', + }, + ], + file: { + type: 'commit_file', + path: 'custom/path/catalog-custom.yaml', + commit: { + repository: { + slug: 'test-repo1', + project: { + // skipped as no match with filter + key: 'project', + }, + mainbranch: { + name: 'main', + }, + links: { + html: { + href: 'https://bitbucket.org/test-ws/test-repo1', + }, + }, + }, + }, + }, + }, + { + path_matches: [ + { + match: true, + text: 'catalog-custom.yaml', + }, + ], + file: { + type: 'commit_file', + path: 'custom/path/catalog-custom.yaml', + commit: { + repository: { + slug: 'test-repo2', + project: { + key: 'test-project', + }, + mainbranch: { + name: 'main', + }, + links: { + html: { + href: 'https://bitbucket.org/test-ws/test-repo2', + }, + }, + }, + }, + }, + }, + ], + }; + return res(ctx.json(response)); + }, + ), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('bitbucketCloud-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://bitbucket.org/test-ws/test-repo2/src/main/custom/path/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + 'bitbucket.org/repo-url': + 'https://bitbucket.org/test-ws/test-repo2', + }, + name: 'generated-7c2e6263b6cc2d14e69fd4d029afba601ad6dc3b', + }, + spec: { + presence: 'required', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'bitbucketCloud-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); + it('update onRepoPush', async () => { const keptModule = createLocationEntity( 'https://bitbucket.org/test-ws/test-repo', diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts index 6826804053..aac5056687 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts @@ -334,7 +334,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { } private async findCatalogFiles( - level: 'workspace' | 'project' = 'workspace', + level: 'workspace' | 'project', repoSlug?: string, ): Promise { const workspace = this.config.workspace; diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts index 1326c53e29..12f8db1c9c 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts @@ -32,7 +32,7 @@ export type BitbucketCloudEntityProviderConfig = { repoSlug?: RegExp; }; schedule?: SchedulerServiceTaskScheduleDefinition; - level?: 'workspace' | 'project'; + level: 'workspace' | 'project'; }; export function readProviderConfigs( @@ -73,6 +73,12 @@ function readProviderConfig( ) : undefined; + const level = + (config.getOptionalString('level') as + | 'workspace' + | 'project' + | undefined) ?? 'workspace'; + return { id, catalogPath, @@ -84,6 +90,7 @@ function readProviderConfig( repoSlug: repoSlugPattern ? compileRegExp(repoSlugPattern) : undefined, }, schedule, + level, }; } From 51fdc5ebf502de39a4ec367b9bd3b8a55cc98c34 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Fri, 27 Sep 2024 12:12:17 +0200 Subject: [PATCH 05/11] test: add config tests Signed-off-by: Benjamin Janssens --- ...BitbucketCloudEntityProviderConfig.test.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts index 7f13616586..0491971ea9 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts @@ -77,13 +77,17 @@ describe('readProviderConfigs', () => { }, }, }, + providerWithProjectLevel: { + workspace: 'test-ws6', + level: 'project', + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(5); + expect(providerConfigs).toHaveLength(6); expect(providerConfigs[0]).toEqual({ id: 'providerWorkspaceOnly', workspace: 'test-ws1', @@ -92,6 +96,7 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: undefined, }, + level: 'workspace', }); expect(providerConfigs[1]).toEqual({ id: 'providerCustomCatalogPath', @@ -101,6 +106,7 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: undefined, }, + level: 'workspace', }); expect(providerConfigs[2]).toEqual({ id: 'providerWithProjectKeyFilter', @@ -110,6 +116,7 @@ describe('readProviderConfigs', () => { projectKey: /^projectKey.*filter$/, repoSlug: undefined, }, + level: 'workspace', }); expect(providerConfigs[3]).toEqual({ id: 'providerWithRepoSlugFilter', @@ -119,6 +126,7 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: /^repoSlug.*filter$/, }, + level: 'workspace', }); expect(providerConfigs[4]).toEqual({ id: 'providerWithSchedule', @@ -134,6 +142,17 @@ describe('readProviderConfigs', () => { minutes: 3, }, }, + level: 'workspace', + }); + expect(providerConfigs[5]).toEqual({ + id: 'providerWithProjectLevel', + workspace: 'test-ws6', + catalogPath: '/catalog-info.yaml', + filters: { + projectKey: undefined, + repoSlug: undefined, + }, + level: 'project', }); }); }); From e07d64016acef6f12427e6e4aebec6117d8b2719 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Fri, 27 Sep 2024 13:37:18 +0200 Subject: [PATCH 06/11] docs: add docs Signed-off-by: Benjamin Janssens --- docs/integrations/bitbucketCloud/discovery.md | 5 +++++ plugins/catalog-backend-module-bitbucket-cloud/config.d.ts | 4 ++-- .../src/providers/BitbucketCloudEntityProvider.test.ts | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/integrations/bitbucketCloud/discovery.md b/docs/integrations/bitbucketCloud/discovery.md index 6ba9c0aa1f..c533f61a2a 100644 --- a/docs/integrations/bitbucketCloud/discovery.md +++ b/docs/integrations/bitbucketCloud/discovery.md @@ -152,6 +152,7 @@ catalog: # supports ISO duration, "human duration" as used in code timeout: { minutes: 3 } workspace: workspace-name + level: workspace # default value ``` > **Note:** It is possible but certainly not recommended to skip the provider ID level. @@ -180,3 +181,7 @@ catalog: - **`workspace`**: Name of your organization account/workspace. If you want to add multiple workspaces, you need to add one provider config each. +- **`level`** _(optional)_: + `'workspace'` (default) or `'project'`. At what level discovery should take place, affecting Bitbucket Cloud API limits. + +> **Note:** By default, discovery will take place at the `workspace` level. While being the most efficient in terms of API calls to Bitbucket Cloud, discovery at the workspace level is limited to 900 repositories per workspace. If your workspace consists of more than 900 repositories, you should switch to discovery at the `project` level, shifting the limit to 900 repositories per project. diff --git a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts index 6ba4cb135d..a56d3cd87a 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts @@ -60,7 +60,7 @@ export interface Config { */ schedule?: SchedulerServiceTaskScheduleDefinitionConfig; /** - * (Optional) On what level discovery should take place, affecting Bitbucket Cloud API limits. + * (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits. * * Possible values: * - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace. @@ -101,7 +101,7 @@ export interface Config { */ schedule?: SchedulerServiceTaskScheduleDefinitionConfig; /** - * (Optional) On what level discovery should take place, affecting Bitbucket Cloud API limits. + * (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits. * * Possible values: * - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace. diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts index f95cb34025..32e8d41551 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts @@ -436,7 +436,7 @@ describe('BitbucketCloudEntityProvider', () => { }); }); - it('apply full update on scheduled execution on project-level', async () => { + it('apply full update on scheduled execution on project level', async () => { const provider = BitbucketCloudEntityProvider.fromConfig( projectLevelConfig, { From f6b4b8a55bc803fbb8f0cecc6ed3d9785268e127 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Fri, 27 Sep 2024 13:39:53 +0200 Subject: [PATCH 07/11] chore: add changeset Signed-off-by: Benjamin Janssens --- .changeset/wise-snakes-sleep.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/wise-snakes-sleep.md diff --git a/.changeset/wise-snakes-sleep.md b/.changeset/wise-snakes-sleep.md new file mode 100644 index 0000000000..f0fa018c9a --- /dev/null +++ b/.changeset/wise-snakes-sleep.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch +--- + +Added discovery level configuration to shift Bitbucket Cloud API limits From d583c32a21c525fc9f0af4afbfebf0a9475bde62 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Tue, 15 Oct 2024 14:58:01 +0200 Subject: [PATCH 08/11] fix: use concat instead of spread operator Signed-off-by: Benjamin Janssens --- .../src/providers/BitbucketCloudEntityProvider.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts index aac5056687..b1a8ac9234 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts @@ -352,12 +352,12 @@ export class BitbucketCloudEntityProvider implements EntityProvider { .listProjectsByWorkspace(workspace) .iterateResults(); - const results: IngestionTarget[] = []; + let results: IngestionTarget[] = []; for await (const project of projects) { const projectQuery = `${query} project:${project.key}`; const result = await this.processQuery(workspace, projectQuery); - results.push(...result); + results = results.concat(result); } return results; From a8ba32421770740252cafaf2cb8464c5a04672c0 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Tue, 15 Oct 2024 15:14:41 +0200 Subject: [PATCH 09/11] style: reorder config property Signed-off-by: Benjamin Janssens --- docs/integrations/bitbucketCloud/discovery.md | 6 +++--- .../config.d.ts | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/integrations/bitbucketCloud/discovery.md b/docs/integrations/bitbucketCloud/discovery.md index c533f61a2a..d623a5d47c 100644 --- a/docs/integrations/bitbucketCloud/discovery.md +++ b/docs/integrations/bitbucketCloud/discovery.md @@ -146,13 +146,13 @@ catalog: filters: # optional projectKey: '^apis-.*$' # optional; RegExp repoSlug: '^service-.*$' # optional; RegExp + level: workspace # default value schedule: # same options as in SchedulerServiceTaskScheduleDefinition # supports cron, ISO duration, "human duration" as used in code frequency: { minutes: 30 } # supports ISO duration, "human duration" as used in code timeout: { minutes: 3 } workspace: workspace-name - level: workspace # default value ``` > **Note:** It is possible but certainly not recommended to skip the provider ID level. @@ -169,6 +169,8 @@ catalog: Regular expression used to filter results based on the project key. - **`repoSlug`** _(optional)_: Regular expression used to filter results based on the repo slug. +- **`level`** _(optional)_: + `'workspace'` (default) or `'project'`. At what level discovery should take place, affecting Bitbucket Cloud API limits. - **`schedule`**: - **`frequency`**: How often you want the task to run. The system does its best to avoid overlapping invocations. @@ -181,7 +183,5 @@ catalog: - **`workspace`**: Name of your organization account/workspace. If you want to add multiple workspaces, you need to add one provider config each. -- **`level`** _(optional)_: - `'workspace'` (default) or `'project'`. At what level discovery should take place, affecting Bitbucket Cloud API limits. > **Note:** By default, discovery will take place at the `workspace` level. While being the most efficient in terms of API calls to Bitbucket Cloud, discovery at the workspace level is limited to 900 repositories per workspace. If your workspace consists of more than 900 repositories, you should switch to discovery at the `project` level, shifting the limit to 900 repositories per project. diff --git a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts index a56d3cd87a..d216efbd8c 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts @@ -55,10 +55,6 @@ export interface Config { */ projectKey?: string; }; - /** - * (Optional) TaskScheduleDefinition for the discovery. - */ - schedule?: SchedulerServiceTaskScheduleDefinitionConfig; /** * (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits. * @@ -67,6 +63,10 @@ export interface Config { * - `project`: 1 API call per project, limited to 900 repositories per project. */ level?: 'workspace' | 'project'; + /** + * (Optional) TaskScheduleDefinition for the discovery. + */ + schedule?: SchedulerServiceTaskScheduleDefinitionConfig; } | { [name: string]: { @@ -96,10 +96,6 @@ export interface Config { */ projectKey?: string; }; - /** - * (Optional) TaskScheduleDefinition for the discovery. - */ - schedule?: SchedulerServiceTaskScheduleDefinitionConfig; /** * (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits. * @@ -108,6 +104,10 @@ export interface Config { * - `project`: 1 API call per project, limited to 900 repositories per project. */ level?: 'workspace' | 'project'; + /** + * (Optional) TaskScheduleDefinition for the discovery. + */ + schedule?: SchedulerServiceTaskScheduleDefinitionConfig; }; }; }; From 7cc909a2c20e943dda9918109b33ff7b3e24bd2f Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Tue, 15 Oct 2024 17:08:16 +0200 Subject: [PATCH 10/11] refactor: remove level config Signed-off-by: Benjamin Janssens --- .changeset/wise-snakes-sleep.md | 2 +- docs/integrations/bitbucketCloud/discovery.md | 5 - .../config.d.ts | 16 -- .../BitbucketCloudEntityProvider.test.ts | 194 ++---------------- .../providers/BitbucketCloudEntityProvider.ts | 27 +-- ...BitbucketCloudEntityProviderConfig.test.ts | 21 +- .../BitbucketCloudEntityProviderConfig.ts | 8 - 7 files changed, 27 insertions(+), 246 deletions(-) diff --git a/.changeset/wise-snakes-sleep.md b/.changeset/wise-snakes-sleep.md index f0fa018c9a..ca51972f57 100644 --- a/.changeset/wise-snakes-sleep.md +++ b/.changeset/wise-snakes-sleep.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch --- -Added discovery level configuration to shift Bitbucket Cloud API limits +Implemented discovery on project-level to shift Bitbucket Cloud API limits diff --git a/docs/integrations/bitbucketCloud/discovery.md b/docs/integrations/bitbucketCloud/discovery.md index d623a5d47c..6ba9c0aa1f 100644 --- a/docs/integrations/bitbucketCloud/discovery.md +++ b/docs/integrations/bitbucketCloud/discovery.md @@ -146,7 +146,6 @@ catalog: filters: # optional projectKey: '^apis-.*$' # optional; RegExp repoSlug: '^service-.*$' # optional; RegExp - level: workspace # default value schedule: # same options as in SchedulerServiceTaskScheduleDefinition # supports cron, ISO duration, "human duration" as used in code frequency: { minutes: 30 } @@ -169,8 +168,6 @@ catalog: Regular expression used to filter results based on the project key. - **`repoSlug`** _(optional)_: Regular expression used to filter results based on the repo slug. -- **`level`** _(optional)_: - `'workspace'` (default) or `'project'`. At what level discovery should take place, affecting Bitbucket Cloud API limits. - **`schedule`**: - **`frequency`**: How often you want the task to run. The system does its best to avoid overlapping invocations. @@ -183,5 +180,3 @@ catalog: - **`workspace`**: Name of your organization account/workspace. If you want to add multiple workspaces, you need to add one provider config each. - -> **Note:** By default, discovery will take place at the `workspace` level. While being the most efficient in terms of API calls to Bitbucket Cloud, discovery at the workspace level is limited to 900 repositories per workspace. If your workspace consists of more than 900 repositories, you should switch to discovery at the `project` level, shifting the limit to 900 repositories per project. diff --git a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts index d216efbd8c..1051976b24 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts @@ -55,14 +55,6 @@ export interface Config { */ projectKey?: string; }; - /** - * (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits. - * - * Possible values: - * - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace. - * - `project`: 1 API call per project, limited to 900 repositories per project. - */ - level?: 'workspace' | 'project'; /** * (Optional) TaskScheduleDefinition for the discovery. */ @@ -96,14 +88,6 @@ export interface Config { */ projectKey?: string; }; - /** - * (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits. - * - * Possible values: - * - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace. - * - `project`: 1 API call per project, limited to 900 repositories per project. - */ - level?: 'workspace' | 'project'; /** * (Optional) TaskScheduleDefinition for the discovery. */ diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts index 32e8d41551..6036fa73f4 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts @@ -87,23 +87,6 @@ describe('BitbucketCloudEntityProvider', () => { }, }, }); - const projectLevelConfig = new ConfigReader({ - catalog: { - providers: { - bitbucketCloud: { - myProvider: { - workspace: 'test-ws', - catalogPath: 'catalog-custom.yaml', - filters: { - projectKey: 'test-.*', - repoSlug: 'test-.*', - }, - level: 'project', - }, - }, - }, - }, - }); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -291,163 +274,6 @@ describe('BitbucketCloudEntityProvider', () => { 'bitbucketCloud-provider:myProvider', ); - server.use( - rest.get( - `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, - (_req, res, ctx) => { - const response = { - values: [ - { - // skipped as empty - path_matches: [], - file: { - type: 'commit_file', - path: 'path/to/ignored/file', - }, - }, - { - path_matches: [ - { - match: true, - text: 'catalog-custom.yaml', - }, - ], - file: { - type: 'commit_file', - path: 'custom/path/catalog-custom.yaml', - commit: { - repository: { - // skipped as no match with filter - slug: 'repo', - project: { - key: 'test-project', - }, - mainbranch: { - name: 'main', - }, - links: { - html: { - href: 'https://bitbucket.org/test-ws/repo', - }, - }, - }, - }, - }, - }, - { - path_matches: [ - { - match: true, - text: 'catalog-custom.yaml', - }, - ], - file: { - type: 'commit_file', - path: 'custom/path/catalog-custom.yaml', - commit: { - repository: { - slug: 'test-repo1', - project: { - // skipped as no match with filter - key: 'project', - }, - mainbranch: { - name: 'main', - }, - links: { - html: { - href: 'https://bitbucket.org/test-ws/test-repo1', - }, - }, - }, - }, - }, - }, - { - path_matches: [ - { - match: true, - text: 'catalog-custom.yaml', - }, - ], - file: { - type: 'commit_file', - path: 'custom/path/catalog-custom.yaml', - commit: { - repository: { - slug: 'test-repo2', - project: { - key: 'test-project', - }, - mainbranch: { - name: 'main', - }, - links: { - html: { - href: 'https://bitbucket.org/test-ws/test-repo2', - }, - }, - }, - }, - }, - }, - ], - }; - return res(ctx.json(response)); - }, - ), - ); - - await provider.connect(entityProviderConnection); - - const taskDef = schedule.getTasks()[0]; - expect(taskDef.id).toEqual('bitbucketCloud-provider:myProvider:refresh'); - await (taskDef.fn as () => Promise)(); - - const url = `https://bitbucket.org/test-ws/test-repo2/src/main/custom/path/catalog-custom.yaml`; - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': `url:${url}`, - 'backstage.io/managed-by-origin-location': `url:${url}`, - 'bitbucket.org/repo-url': - 'https://bitbucket.org/test-ws/test-repo2', - }, - name: 'generated-7c2e6263b6cc2d14e69fd4d029afba601ad6dc3b', - }, - spec: { - presence: 'required', - target: `${url}`, - type: 'url', - }, - }, - locationKey: 'bitbucketCloud-provider:myProvider', - }, - ]; - - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); - expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'full', - entities: expectedEntities, - }); - }); - - it('apply full update on scheduled execution on project level', async () => { - const provider = BitbucketCloudEntityProvider.fromConfig( - projectLevelConfig, - { - logger, - schedule, - }, - )[0]; - expect(provider.getProviderName()).toEqual( - 'bitbucketCloud-provider:myProvider', - ); - server.use( rest.get( `https://api.bitbucket.org/2.0/workspaces/test-ws/projects`, @@ -464,12 +290,7 @@ describe('BitbucketCloudEntityProvider', () => { ), rest.get( `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, - (req, res, ctx) => { - const query = req.url.searchParams.get('search_query'); - if (!query || !query.includes('project:TEST')) { - return res(ctx.json({ values: [] })); - } - + (_req, res, ctx) => { const response = { values: [ { @@ -657,6 +478,19 @@ describe('BitbucketCloudEntityProvider', () => { })[0]; server.use( + rest.get( + `https://api.bitbucket.org/2.0/workspaces/test-ws/projects`, + (_req, res, ctx) => { + const response = { + values: [ + { + key: 'TEST', + }, + ], + }; + return res(ctx.json(response)); + }, + ), rest.get( `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, (req, res, ctx) => { diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts index b1a8ac9234..891ad14da5 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts @@ -207,7 +207,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { logger.info('Discovering catalog files in Bitbucket Cloud repositories'); - const targets = await this.findCatalogFiles(this.config.level); + const targets = await this.findCatalogFiles(); const entities = this.toDeferredEntities(targets); await this.connection.applyMutation({ @@ -271,7 +271,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { // Hence, we will just trigger a refresh for catalog file(s) within the repository // if we get notified about changes there. - const targets = await this.findCatalogFiles('workspace', repoSlug); + const targets = await this.findCatalogFiles(repoSlug); const { token } = await this.tokenManager!.getToken(); const existing = await this.findExistingLocations(repoUrl, token); @@ -334,7 +334,6 @@ export class BitbucketCloudEntityProvider implements EntityProvider { } private async findCatalogFiles( - level: 'workspace' | 'project', repoSlug?: string, ): Promise { const workspace = this.config.workspace; @@ -347,23 +346,19 @@ export class BitbucketCloudEntityProvider implements EntityProvider { const optRepoFilter = repoSlug ? ` repo:${repoSlug}` : ''; const query = `"${catalogFilename}" path:${catalogPath}${optRepoFilter}`; - if (level === 'project') { - const projects = this.client - .listProjectsByWorkspace(workspace) - .iterateResults(); + const projects = this.client + .listProjectsByWorkspace(workspace) + .iterateResults(); - let results: IngestionTarget[] = []; + let results: IngestionTarget[] = []; - for await (const project of projects) { - const projectQuery = `${query} project:${project.key}`; - const result = await this.processQuery(workspace, projectQuery); - results = results.concat(result); - } - - return results; + for await (const project of projects) { + const projectQuery = `${query} project:${project.key}`; + const result = await this.processQuery(workspace, projectQuery); + results = results.concat(result); } - return this.processQuery(workspace, query); + return results; } private async processQuery( diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts index 0491971ea9..7f13616586 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts @@ -77,17 +77,13 @@ describe('readProviderConfigs', () => { }, }, }, - providerWithProjectLevel: { - workspace: 'test-ws6', - level: 'project', - }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(6); + expect(providerConfigs).toHaveLength(5); expect(providerConfigs[0]).toEqual({ id: 'providerWorkspaceOnly', workspace: 'test-ws1', @@ -96,7 +92,6 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: undefined, }, - level: 'workspace', }); expect(providerConfigs[1]).toEqual({ id: 'providerCustomCatalogPath', @@ -106,7 +101,6 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: undefined, }, - level: 'workspace', }); expect(providerConfigs[2]).toEqual({ id: 'providerWithProjectKeyFilter', @@ -116,7 +110,6 @@ describe('readProviderConfigs', () => { projectKey: /^projectKey.*filter$/, repoSlug: undefined, }, - level: 'workspace', }); expect(providerConfigs[3]).toEqual({ id: 'providerWithRepoSlugFilter', @@ -126,7 +119,6 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: /^repoSlug.*filter$/, }, - level: 'workspace', }); expect(providerConfigs[4]).toEqual({ id: 'providerWithSchedule', @@ -142,17 +134,6 @@ describe('readProviderConfigs', () => { minutes: 3, }, }, - level: 'workspace', - }); - expect(providerConfigs[5]).toEqual({ - id: 'providerWithProjectLevel', - workspace: 'test-ws6', - catalogPath: '/catalog-info.yaml', - filters: { - projectKey: undefined, - repoSlug: undefined, - }, - level: 'project', }); }); }); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts index 12f8db1c9c..5a806d2e0b 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts @@ -32,7 +32,6 @@ export type BitbucketCloudEntityProviderConfig = { repoSlug?: RegExp; }; schedule?: SchedulerServiceTaskScheduleDefinition; - level: 'workspace' | 'project'; }; export function readProviderConfigs( @@ -73,12 +72,6 @@ function readProviderConfig( ) : undefined; - const level = - (config.getOptionalString('level') as - | 'workspace' - | 'project' - | undefined) ?? 'workspace'; - return { id, catalogPath, @@ -90,7 +83,6 @@ function readProviderConfig( repoSlug: repoSlugPattern ? compileRegExp(repoSlugPattern) : undefined, }, schedule, - level, }; } From 5838d47e54a293c978a90cad8c3e8e0041f50d1c Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Fri, 25 Oct 2024 11:18:41 +0200 Subject: [PATCH 11/11] test: use extra project in tests Signed-off-by: Benjamin Janssens --- .../BitbucketCloudEntityProvider.test.ts | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts index 4d5103a718..fe460b8d8b 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts @@ -320,6 +320,9 @@ describe('BitbucketCloudEntityProvider', () => { { key: 'TEST', }, + { + key: 'TEST2', + }, ], }; return res(ctx.json(response)); @@ -460,6 +463,27 @@ describe('BitbucketCloudEntityProvider', () => { }, locationKey: 'bitbucketCloud-provider:myProvider', }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + 'bitbucket.org/repo-url': + 'https://bitbucket.org/test-ws/test-repo2', + }, + name: 'generated-7c2e6263b6cc2d14e69fd4d029afba601ad6dc3b', + }, + spec: { + presence: 'required', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'bitbucketCloud-provider:myProvider', + }, ]; expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); @@ -526,6 +550,9 @@ describe('BitbucketCloudEntityProvider', () => { { key: 'TEST', }, + { + key: 'TEST2', + }, ], }; return res(ctx.json(response)); @@ -535,7 +562,11 @@ describe('BitbucketCloudEntityProvider', () => { `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, (req, res, ctx) => { const query = req.url.searchParams.get('search_query'); - if (!query || !query.includes('repo:test-repo')) { + if ( + !query || + !query.includes('repo:test-repo') || + !query.includes('project:TEST') + ) { return res(ctx.json({ values: [] })); } @@ -612,6 +643,10 @@ describe('BitbucketCloudEntityProvider', () => { entity: addedModule, locationKey: 'bitbucketCloud-provider:myProvider', }, + { + entity: addedModule, + locationKey: 'bitbucketCloud-provider:myProvider', + }, ]; const removedEntities = [ {