diff --git a/.changeset/fifty-sides-begin.md b/.changeset/fifty-sides-begin.md new file mode 100644 index 0000000000..5cd174e4ff --- /dev/null +++ b/.changeset/fifty-sides-begin.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-server': minor +--- + +Add validateLocationsExist option to avoid generating locations for catalog-info.yaml files that do not exist in the source repository. diff --git a/docs/integrations/bitbucketServer/discovery.md b/docs/integrations/bitbucketServer/discovery.md index 29c8d40338..350bc33aed 100644 --- a/docs/integrations/bitbucketServer/discovery.md +++ b/docs/integrations/bitbucketServer/discovery.md @@ -66,6 +66,7 @@ catalog: projectKey: '^apis-.*$' # optional; RegExp repoSlug: '^service-.*$' # optional; RegExp skipArchivedRepos: true # optional; boolean + validateLocationsExist: false # optional; boolean schedule: # same options as in SchedulerServiceTaskScheduleDefinition # supports cron, ISO duration, "human duration" as used in code frequency: { minutes: 30 } @@ -86,6 +87,10 @@ catalog: Regular expression used to filter results based on the repo slug. - **`skipArchivedRepos`** _(optional)_: Boolean flag to filter out archived repositories. +- **`validateLocationsExist`** _(optional)_: + Defaults to `false`. + Whether to validate locations that exist before emitting them. + This option avoids generating locations for catalog info files that do not exist in the source repository. - **`schedule`**: - **`frequency`**: How often you want the task to run. The system does its best to avoid overlapping invocations. diff --git a/plugins/catalog-backend-module-bitbucket-server/config.d.ts b/plugins/catalog-backend-module-bitbucket-server/config.d.ts index 0f6889efa7..d57a15a736 100644 --- a/plugins/catalog-backend-module-bitbucket-server/config.d.ts +++ b/plugins/catalog-backend-module-bitbucket-server/config.d.ts @@ -51,6 +51,11 @@ export interface Config { */ skipArchivedRepos?: boolean; }; + /** + * (Optional) Whether to validate locations that exist before emitting them. + * Default: `false`. + */ + validateLocationsExist?: boolean; /** * (Optional) TaskScheduleDefinition for the refresh. */ @@ -83,6 +88,11 @@ export interface Config { */ skipArchivedRepos?: boolean; }; + /** + * (Optional) Whether to validate locations that exist before emitting them. + * Default: `false`. + */ + validateLocationsExist?: boolean; /** * (Optional) TaskScheduleDefinition for the refresh. */ diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts index fa0c455b04..7bf6480f33 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts @@ -70,8 +70,11 @@ export class BitbucketServerClient { repo: string; path: string; }): Promise { + const normalizedPath = options.path.startsWith('/') + ? options.path.substring(1) + : options.path; return fetch( - `${this.config.apiBaseUrl}/projects/${options.projectKey}/repos/${options.repo}/raw/${options.path}`, + `${this.config.apiBaseUrl}/projects/${options.projectKey}/repos/${options.repo}/raw/${normalizedPath}`, getBitbucketServerRequestOptions(this.config), ); } diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts index ba4e3e0411..790bef7939 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts @@ -668,6 +668,97 @@ describe('BitbucketServerEntityProvider', () => { }); }); + it('do not add locations when validateLocationsExist and catalog-info does not exist', async () => { + server.use( + rest.get( + `https://${host}/rest/api/1.0/projects/project-test/repos/repo-test/raw/catalog-info.yaml`, + (_, res, ctx) => { + return res(ctx.status(404)); + }, + ), + rest.get( + `https://${host}/rest/api/1.0/projects/other-project/repos/other-repo/raw/catalog-info.yaml`, + (_, res, ctx) => { + return res(ctx.status(200)); + }, + ), + ); + const config = new ConfigReader({ + integrations: { + bitbucketServer: [ + { + host: host, + }, + ], + }, + catalog: { + providers: { + bitbucketServer: { + mainProvider: { + host: host, + validateLocationsExist: true, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = BitbucketServerEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'bitbucketServer-provider:mainProvider', + ); + + setupStubs( + [ + { key: 'project-test', repos: [{ name: 'repo-test' }] }, + { key: 'other-project', repos: [{ name: 'other-repo' }] }, + ], + `https://${host}`, + 'master', + ); + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('bitbucketServer-provider:mainProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + 'backstage.io/managed-by-origin-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + 'bitbucket.org/default-branch': 'master', + }, + name: 'generated-d8d4944c30c2906dfee172ddda9537f9893b2c0f', + }, + spec: { + presence: 'optional', + target: `https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + type: 'url', + }, + }, + locationKey: 'bitbucketServer-provider:mainProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); + it('Multiple location entities to deferred entities', async () => { const schedule = new PersistingTaskRunner(); const config = new ConfigReader({ diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts index b462f8756a..4b02632892 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts @@ -253,6 +253,37 @@ export class BitbucketServerEntityProvider implements EntityProvider { if (this.config?.filters?.skipArchivedRepos && repository.archived) { continue; } + if (this.config.validateLocationsExist) { + try { + const response = await client.getFile({ + projectKey: project.key, + repo: repository.slug, + path: this.config.catalogPath, + }); + if (!response.ok) { + if (response.status === 404) { + this.logger.debug( + `Skipping repository ${repository.slug} in project ${project.key} because the catalog file does not exist.`, + ); + } else { + this.logger.warn( + `Unexpected response code ${ + response.status + } while fetching the catalog file from repository ${ + repository.slug + } in project ${ + project.key + }. Response details: ${JSON.stringify(response)}`, + ); + } + continue; + } + } catch (error) { + this.logger.error( + `An error occurred while fetching the catalog file from repository ${repository.slug} in project ${project.key}: ${error}`, + ); + } + } for await (const entity of this.parser({ client, logger: this.logger, diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts index 755fa4b58a..cd39f0659b 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts @@ -49,6 +49,8 @@ describe('readProviderConfigs', () => { repoSlug: undefined, skipArchivedRepos: undefined, }, + schedule: undefined, + validateLocationsExist: false, }); }); @@ -89,6 +91,8 @@ describe('readProviderConfigs', () => { repoSlug: undefined, skipArchivedRepos: undefined, }, + schedule: undefined, + validateLocationsExist: false, }); expect(providerConfigs[1]).toEqual({ id: 'secondaryProvider', @@ -99,6 +103,8 @@ describe('readProviderConfigs', () => { repoSlug: undefined, skipArchivedRepos: undefined, }, + schedule: undefined, + validateLocationsExist: false, }); expect(providerConfigs[2]).toEqual({ id: 'thirdProvider', @@ -109,6 +115,7 @@ describe('readProviderConfigs', () => { repoSlug: undefined, skipArchivedRepos: undefined, }, + validateLocationsExist: false, schedule: { frequency: { minutes: 30 }, timeout: { @@ -147,6 +154,36 @@ describe('readProviderConfigs', () => { repoSlug: /.*/, skipArchivedRepos: true, }, + schedule: undefined, + validateLocationsExist: false, + }); + }); + + it('single simple provider config with validateLocationsExist', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + host: 'bitbucket.mycompany.com', + validateLocationsExist: true, + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0]).toEqual({ + id: 'default', + catalogPath: '/catalog-info.yaml', + host: 'bitbucket.mycompany.com', + filters: { + projectKey: undefined, + repoSlug: undefined, + skipArchivedRepos: undefined, + }, + schedule: undefined, + validateLocationsExist: true, }); }); }); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts index 3b921b6bab..1a224536a7 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts @@ -32,6 +32,7 @@ export type BitbucketServerEntityProviderConfig = { repoSlug?: RegExp; skipArchivedRepos?: boolean; }; + validateLocationsExist: boolean; schedule?: SchedulerServiceTaskScheduleDefinition; }; @@ -66,7 +67,8 @@ function readProviderConfig( const skipArchivedReposFlag = config.getOptionalBoolean( 'filters.skipArchivedRepos', ); - + const validateLocationsExistFlag = + config?.getOptionalBoolean('validateLocationsExist') ?? false; const schedule = config.has('schedule') ? readSchedulerServiceTaskScheduleDefinitionFromConfig( config.getConfig('schedule'), @@ -82,6 +84,7 @@ function readProviderConfig( repoSlug: repoSlugPattern ? new RegExp(repoSlugPattern) : undefined, skipArchivedRepos: skipArchivedReposFlag, }, + validateLocationsExist: validateLocationsExistFlag, schedule, }; }