From 7cc909a2c20e943dda9918109b33ff7b3e24bd2f Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Tue, 15 Oct 2024 17:08:16 +0200 Subject: [PATCH] refactor: remove level config Signed-off-by: Benjamin Janssens --- .changeset/wise-snakes-sleep.md | 2 +- docs/integrations/bitbucketCloud/discovery.md | 5 - .../config.d.ts | 16 -- .../BitbucketCloudEntityProvider.test.ts | 194 ++---------------- .../providers/BitbucketCloudEntityProvider.ts | 27 +-- ...BitbucketCloudEntityProviderConfig.test.ts | 21 +- .../BitbucketCloudEntityProviderConfig.ts | 8 - 7 files changed, 27 insertions(+), 246 deletions(-) diff --git a/.changeset/wise-snakes-sleep.md b/.changeset/wise-snakes-sleep.md index f0fa018c9a..ca51972f57 100644 --- a/.changeset/wise-snakes-sleep.md +++ b/.changeset/wise-snakes-sleep.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch --- -Added discovery level configuration to shift Bitbucket Cloud API limits +Implemented discovery on project-level to shift Bitbucket Cloud API limits diff --git a/docs/integrations/bitbucketCloud/discovery.md b/docs/integrations/bitbucketCloud/discovery.md index d623a5d47c..6ba9c0aa1f 100644 --- a/docs/integrations/bitbucketCloud/discovery.md +++ b/docs/integrations/bitbucketCloud/discovery.md @@ -146,7 +146,6 @@ catalog: filters: # optional projectKey: '^apis-.*$' # optional; RegExp repoSlug: '^service-.*$' # optional; RegExp - level: workspace # default value schedule: # same options as in SchedulerServiceTaskScheduleDefinition # supports cron, ISO duration, "human duration" as used in code frequency: { minutes: 30 } @@ -169,8 +168,6 @@ catalog: Regular expression used to filter results based on the project key. - **`repoSlug`** _(optional)_: Regular expression used to filter results based on the repo slug. -- **`level`** _(optional)_: - `'workspace'` (default) or `'project'`. At what level discovery should take place, affecting Bitbucket Cloud API limits. - **`schedule`**: - **`frequency`**: How often you want the task to run. The system does its best to avoid overlapping invocations. @@ -183,5 +180,3 @@ catalog: - **`workspace`**: Name of your organization account/workspace. If you want to add multiple workspaces, you need to add one provider config each. - -> **Note:** By default, discovery will take place at the `workspace` level. While being the most efficient in terms of API calls to Bitbucket Cloud, discovery at the workspace level is limited to 900 repositories per workspace. If your workspace consists of more than 900 repositories, you should switch to discovery at the `project` level, shifting the limit to 900 repositories per project. diff --git a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts index d216efbd8c..1051976b24 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts @@ -55,14 +55,6 @@ export interface Config { */ projectKey?: string; }; - /** - * (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits. - * - * Possible values: - * - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace. - * - `project`: 1 API call per project, limited to 900 repositories per project. - */ - level?: 'workspace' | 'project'; /** * (Optional) TaskScheduleDefinition for the discovery. */ @@ -96,14 +88,6 @@ export interface Config { */ projectKey?: string; }; - /** - * (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits. - * - * Possible values: - * - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace. - * - `project`: 1 API call per project, limited to 900 repositories per project. - */ - level?: 'workspace' | 'project'; /** * (Optional) TaskScheduleDefinition for the discovery. */ diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts index 32e8d41551..6036fa73f4 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts @@ -87,23 +87,6 @@ describe('BitbucketCloudEntityProvider', () => { }, }, }); - const projectLevelConfig = new ConfigReader({ - catalog: { - providers: { - bitbucketCloud: { - myProvider: { - workspace: 'test-ws', - catalogPath: 'catalog-custom.yaml', - filters: { - projectKey: 'test-.*', - repoSlug: 'test-.*', - }, - level: 'project', - }, - }, - }, - }, - }); const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), @@ -291,163 +274,6 @@ describe('BitbucketCloudEntityProvider', () => { 'bitbucketCloud-provider:myProvider', ); - server.use( - rest.get( - `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, - (_req, res, ctx) => { - const response = { - values: [ - { - // skipped as empty - path_matches: [], - file: { - type: 'commit_file', - path: 'path/to/ignored/file', - }, - }, - { - path_matches: [ - { - match: true, - text: 'catalog-custom.yaml', - }, - ], - file: { - type: 'commit_file', - path: 'custom/path/catalog-custom.yaml', - commit: { - repository: { - // skipped as no match with filter - slug: 'repo', - project: { - key: 'test-project', - }, - mainbranch: { - name: 'main', - }, - links: { - html: { - href: 'https://bitbucket.org/test-ws/repo', - }, - }, - }, - }, - }, - }, - { - path_matches: [ - { - match: true, - text: 'catalog-custom.yaml', - }, - ], - file: { - type: 'commit_file', - path: 'custom/path/catalog-custom.yaml', - commit: { - repository: { - slug: 'test-repo1', - project: { - // skipped as no match with filter - key: 'project', - }, - mainbranch: { - name: 'main', - }, - links: { - html: { - href: 'https://bitbucket.org/test-ws/test-repo1', - }, - }, - }, - }, - }, - }, - { - path_matches: [ - { - match: true, - text: 'catalog-custom.yaml', - }, - ], - file: { - type: 'commit_file', - path: 'custom/path/catalog-custom.yaml', - commit: { - repository: { - slug: 'test-repo2', - project: { - key: 'test-project', - }, - mainbranch: { - name: 'main', - }, - links: { - html: { - href: 'https://bitbucket.org/test-ws/test-repo2', - }, - }, - }, - }, - }, - }, - ], - }; - return res(ctx.json(response)); - }, - ), - ); - - await provider.connect(entityProviderConnection); - - const taskDef = schedule.getTasks()[0]; - expect(taskDef.id).toEqual('bitbucketCloud-provider:myProvider:refresh'); - await (taskDef.fn as () => Promise)(); - - const url = `https://bitbucket.org/test-ws/test-repo2/src/main/custom/path/catalog-custom.yaml`; - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': `url:${url}`, - 'backstage.io/managed-by-origin-location': `url:${url}`, - 'bitbucket.org/repo-url': - 'https://bitbucket.org/test-ws/test-repo2', - }, - name: 'generated-7c2e6263b6cc2d14e69fd4d029afba601ad6dc3b', - }, - spec: { - presence: 'required', - target: `${url}`, - type: 'url', - }, - }, - locationKey: 'bitbucketCloud-provider:myProvider', - }, - ]; - - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); - expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'full', - entities: expectedEntities, - }); - }); - - it('apply full update on scheduled execution on project level', async () => { - const provider = BitbucketCloudEntityProvider.fromConfig( - projectLevelConfig, - { - logger, - schedule, - }, - )[0]; - expect(provider.getProviderName()).toEqual( - 'bitbucketCloud-provider:myProvider', - ); - server.use( rest.get( `https://api.bitbucket.org/2.0/workspaces/test-ws/projects`, @@ -464,12 +290,7 @@ describe('BitbucketCloudEntityProvider', () => { ), rest.get( `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, - (req, res, ctx) => { - const query = req.url.searchParams.get('search_query'); - if (!query || !query.includes('project:TEST')) { - return res(ctx.json({ values: [] })); - } - + (_req, res, ctx) => { const response = { values: [ { @@ -657,6 +478,19 @@ describe('BitbucketCloudEntityProvider', () => { })[0]; server.use( + rest.get( + `https://api.bitbucket.org/2.0/workspaces/test-ws/projects`, + (_req, res, ctx) => { + const response = { + values: [ + { + key: 'TEST', + }, + ], + }; + return res(ctx.json(response)); + }, + ), rest.get( `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, (req, res, ctx) => { diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts index b1a8ac9234..891ad14da5 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts @@ -207,7 +207,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { logger.info('Discovering catalog files in Bitbucket Cloud repositories'); - const targets = await this.findCatalogFiles(this.config.level); + const targets = await this.findCatalogFiles(); const entities = this.toDeferredEntities(targets); await this.connection.applyMutation({ @@ -271,7 +271,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { // Hence, we will just trigger a refresh for catalog file(s) within the repository // if we get notified about changes there. - const targets = await this.findCatalogFiles('workspace', repoSlug); + const targets = await this.findCatalogFiles(repoSlug); const { token } = await this.tokenManager!.getToken(); const existing = await this.findExistingLocations(repoUrl, token); @@ -334,7 +334,6 @@ export class BitbucketCloudEntityProvider implements EntityProvider { } private async findCatalogFiles( - level: 'workspace' | 'project', repoSlug?: string, ): Promise { const workspace = this.config.workspace; @@ -347,23 +346,19 @@ export class BitbucketCloudEntityProvider implements EntityProvider { const optRepoFilter = repoSlug ? ` repo:${repoSlug}` : ''; const query = `"${catalogFilename}" path:${catalogPath}${optRepoFilter}`; - if (level === 'project') { - const projects = this.client - .listProjectsByWorkspace(workspace) - .iterateResults(); + const projects = this.client + .listProjectsByWorkspace(workspace) + .iterateResults(); - let results: IngestionTarget[] = []; + let results: IngestionTarget[] = []; - for await (const project of projects) { - const projectQuery = `${query} project:${project.key}`; - const result = await this.processQuery(workspace, projectQuery); - results = results.concat(result); - } - - return results; + for await (const project of projects) { + const projectQuery = `${query} project:${project.key}`; + const result = await this.processQuery(workspace, projectQuery); + results = results.concat(result); } - return this.processQuery(workspace, query); + return results; } private async processQuery( diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts index 0491971ea9..7f13616586 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts @@ -77,17 +77,13 @@ describe('readProviderConfigs', () => { }, }, }, - providerWithProjectLevel: { - workspace: 'test-ws6', - level: 'project', - }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(6); + expect(providerConfigs).toHaveLength(5); expect(providerConfigs[0]).toEqual({ id: 'providerWorkspaceOnly', workspace: 'test-ws1', @@ -96,7 +92,6 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: undefined, }, - level: 'workspace', }); expect(providerConfigs[1]).toEqual({ id: 'providerCustomCatalogPath', @@ -106,7 +101,6 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: undefined, }, - level: 'workspace', }); expect(providerConfigs[2]).toEqual({ id: 'providerWithProjectKeyFilter', @@ -116,7 +110,6 @@ describe('readProviderConfigs', () => { projectKey: /^projectKey.*filter$/, repoSlug: undefined, }, - level: 'workspace', }); expect(providerConfigs[3]).toEqual({ id: 'providerWithRepoSlugFilter', @@ -126,7 +119,6 @@ describe('readProviderConfigs', () => { projectKey: undefined, repoSlug: /^repoSlug.*filter$/, }, - level: 'workspace', }); expect(providerConfigs[4]).toEqual({ id: 'providerWithSchedule', @@ -142,17 +134,6 @@ describe('readProviderConfigs', () => { minutes: 3, }, }, - level: 'workspace', - }); - expect(providerConfigs[5]).toEqual({ - id: 'providerWithProjectLevel', - workspace: 'test-ws6', - catalogPath: '/catalog-info.yaml', - filters: { - projectKey: undefined, - repoSlug: undefined, - }, - level: 'project', }); }); }); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts index 12f8db1c9c..5a806d2e0b 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts @@ -32,7 +32,6 @@ export type BitbucketCloudEntityProviderConfig = { repoSlug?: RegExp; }; schedule?: SchedulerServiceTaskScheduleDefinition; - level: 'workspace' | 'project'; }; export function readProviderConfigs( @@ -73,12 +72,6 @@ function readProviderConfig( ) : undefined; - const level = - (config.getOptionalString('level') as - | 'workspace' - | 'project' - | undefined) ?? 'workspace'; - return { id, catalogPath, @@ -90,7 +83,6 @@ function readProviderConfig( repoSlug: repoSlugPattern ? compileRegExp(repoSlugPattern) : undefined, }, schedule, - level, }; }