Merge pull request #30219 from xoanmi/master

Add validateLocationsExist option for the BitbucketServer entity provider

.changeset/fifty-sides-begin.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-bitbucket-server': minor
+---
+
+Add validateLocationsExist option to avoid generating locations for catalog-info.yaml files that do not exist in the source repository.

docs/integrations/bitbucketServer/discovery.md

@@ -66,6 +66,7 @@ catalog:
           projectKey: '^apis-.*$' # optional; RegExp
           repoSlug: '^service-.*$' # optional; RegExp
           skipArchivedRepos: true # optional; boolean
+        validateLocationsExist: false # optional; boolean
         schedule: # same options as in SchedulerServiceTaskScheduleDefinition
           # supports cron, ISO duration, "human duration" as used in code
           frequency: { minutes: 30 }
@@ -86,6 +87,10 @@ catalog:
     Regular expression used to filter results based on the repo slug.
   - **`skipArchivedRepos`** _(optional)_:
     Boolean flag to filter out archived repositories.
+- **`validateLocationsExist`** _(optional)_:
+  Defaults to `false`.
+  Whether to validate locations that exist before emitting them.
+  This option avoids generating locations for catalog info files that do not exist in the source repository.
 - **`schedule`**:
   - **`frequency`**:
     How often you want the task to run. The system does its best to avoid overlapping invocations.

plugins/catalog-backend-module-bitbucket-server/config.d.ts

@@ -51,6 +51,11 @@ export interface Config {
                */
               skipArchivedRepos?: boolean;
             };
+            /**
+             * (Optional) Whether to validate locations that exist before emitting them.
+             * Default: `false`.
+             */
+            validateLocationsExist?: boolean;
             /**
              * (Optional) TaskScheduleDefinition for the refresh.
              */
@@ -83,6 +88,11 @@ export interface Config {
                  */
                 skipArchivedRepos?: boolean;
               };
+              /**
+               * (Optional) Whether to validate locations that exist before emitting them.
+               * Default: `false`.
+               */
+              validateLocationsExist?: boolean;
               /**
                * (Optional) TaskScheduleDefinition for the refresh.
                */

plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts

@@ -70,8 +70,11 @@ export class BitbucketServerClient {
     repo: string;
     path: string;
   }): Promise<Response> {
+    const normalizedPath = options.path.startsWith('/')
+      ? options.path.substring(1)
+      : options.path;
     return fetch(
-      `${this.config.apiBaseUrl}/projects/${options.projectKey}/repos/${options.repo}/raw/${options.path}`,
+      `${this.config.apiBaseUrl}/projects/${options.projectKey}/repos/${options.repo}/raw/${normalizedPath}`,
       getBitbucketServerRequestOptions(this.config),
     );
   }

plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts

@@ -668,6 +668,97 @@ describe('BitbucketServerEntityProvider', () => {
     });
   });
 
+  it('do not add locations when validateLocationsExist and catalog-info does not exist', async () => {
+    server.use(
+      rest.get(
+        `https://${host}/rest/api/1.0/projects/project-test/repos/repo-test/raw/catalog-info.yaml`,
+        (_, res, ctx) => {
+          return res(ctx.status(404));
+        },
+      ),
+      rest.get(
+        `https://${host}/rest/api/1.0/projects/other-project/repos/other-repo/raw/catalog-info.yaml`,
+        (_, res, ctx) => {
+          return res(ctx.status(200));
+        },
+      ),
+    );
+    const config = new ConfigReader({
+      integrations: {
+        bitbucketServer: [
+          {
+            host: host,
+          },
+        ],
+      },
+      catalog: {
+        providers: {
+          bitbucketServer: {
+            mainProvider: {
+              host: host,
+              validateLocationsExist: true,
+            },
+          },
+        },
+      },
+    });
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = BitbucketServerEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+    expect(provider.getProviderName()).toEqual(
+      'bitbucketServer-provider:mainProvider',
+    );
+
+    setupStubs(
+      [
+        { key: 'project-test', repos: [{ name: 'repo-test' }] },
+        { key: 'other-project', repos: [{ name: 'other-repo' }] },
+      ],
+      `https://${host}`,
+      'master',
+    );
+    await provider.connect(entityProviderConnection);
+
+    const taskDef = schedule.getTasks()[0];
+    expect(taskDef.id).toEqual('bitbucketServer-provider:mainProvider:refresh');
+    await (taskDef.fn as () => Promise<void>)();
+
+    const expectedEntities = [
+      {
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Location',
+          metadata: {
+            annotations: {
+              'backstage.io/managed-by-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`,
+              'backstage.io/managed-by-origin-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`,
+              'bitbucket.org/default-branch': 'master',
+            },
+            name: 'generated-d8d4944c30c2906dfee172ddda9537f9893b2c0f',
+          },
+          spec: {
+            presence: 'optional',
+            target: `https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`,
+            type: 'url',
+          },
+        },
+        locationKey: 'bitbucketServer-provider:mainProvider',
+      },
+    ];
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: expectedEntities,
+    });
+  });
+
   it('Multiple location entities to deferred entities', async () => {
     const schedule = new PersistingTaskRunner();
     const config = new ConfigReader({

plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts

@@ -253,6 +253,37 @@ export class BitbucketServerEntityProvider implements EntityProvider {
         if (this.config?.filters?.skipArchivedRepos && repository.archived) {
           continue;
         }
+        if (this.config.validateLocationsExist) {
+          try {
+            const response = await client.getFile({
+              projectKey: project.key,
+              repo: repository.slug,
+              path: this.config.catalogPath,
+            });
+            if (!response.ok) {
+              if (response.status === 404) {
+                this.logger.debug(
+                  `Skipping repository ${repository.slug} in project ${project.key} because the catalog file does not exist.`,
+                );
+              } else {
+                this.logger.warn(
+                  `Unexpected response code ${
+                    response.status
+                  } while fetching the catalog file from repository ${
+                    repository.slug
+                  } in project ${
+                    project.key
+                  }. Response details: ${JSON.stringify(response)}`,
+                );
+              }
+              continue;
+            }
+          } catch (error) {
+            this.logger.error(
+              `An error occurred while fetching the catalog file from repository ${repository.slug} in project ${project.key}: ${error}`,
+            );
+          }
+        }
         for await (const entity of this.parser({
           client,
           logger: this.logger,

plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts

@@ -49,6 +49,8 @@ describe('readProviderConfigs', () => {
         repoSlug: undefined,
         skipArchivedRepos: undefined,
       },
+      schedule: undefined,
+      validateLocationsExist: false,
     });
   });
 
@@ -89,6 +91,8 @@ describe('readProviderConfigs', () => {
         repoSlug: undefined,
         skipArchivedRepos: undefined,
       },
+      schedule: undefined,
+      validateLocationsExist: false,
     });
     expect(providerConfigs[1]).toEqual({
       id: 'secondaryProvider',
@@ -99,6 +103,8 @@ describe('readProviderConfigs', () => {
         repoSlug: undefined,
         skipArchivedRepos: undefined,
       },
+      schedule: undefined,
+      validateLocationsExist: false,
     });
     expect(providerConfigs[2]).toEqual({
       id: 'thirdProvider',
@@ -109,6 +115,7 @@ describe('readProviderConfigs', () => {
         repoSlug: undefined,
         skipArchivedRepos: undefined,
       },
+      validateLocationsExist: false,
       schedule: {
         frequency: { minutes: 30 },
         timeout: {
@@ -147,6 +154,36 @@ describe('readProviderConfigs', () => {
         repoSlug: /.*/,
         skipArchivedRepos: true,
       },
+      schedule: undefined,
+      validateLocationsExist: false,
+    });
+  });
+
+  it('single simple provider config with validateLocationsExist', () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          bitbucketServer: {
+            host: 'bitbucket.mycompany.com',
+            validateLocationsExist: true,
+          },
+        },
+      },
+    });
+    const providerConfigs = readProviderConfigs(config);
+
+    expect(providerConfigs).toHaveLength(1);
+    expect(providerConfigs[0]).toEqual({
+      id: 'default',
+      catalogPath: '/catalog-info.yaml',
+      host: 'bitbucket.mycompany.com',
+      filters: {
+        projectKey: undefined,
+        repoSlug: undefined,
+        skipArchivedRepos: undefined,
+      },
+      schedule: undefined,
+      validateLocationsExist: true,
     });
   });
 });

plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts

@@ -32,6 +32,7 @@ export type BitbucketServerEntityProviderConfig = {
     repoSlug?: RegExp;
     skipArchivedRepos?: boolean;
   };
+  validateLocationsExist: boolean;
   schedule?: SchedulerServiceTaskScheduleDefinition;
 };
 
@@ -66,7 +67,8 @@ function readProviderConfig(
   const skipArchivedReposFlag = config.getOptionalBoolean(
     'filters.skipArchivedRepos',
   );
-
+  const validateLocationsExistFlag =
+    config?.getOptionalBoolean('validateLocationsExist') ?? false;
   const schedule = config.has('schedule')
     ? readSchedulerServiceTaskScheduleDefinitionFromConfig(
         config.getConfig('schedule'),
@@ -82,6 +84,7 @@ function readProviderConfig(
       repoSlug: repoSlugPattern ? new RegExp(repoSlugPattern) : undefined,
       skipArchivedRepos: skipArchivedReposFlag,
     },
+    validateLocationsExist: validateLocationsExistFlag,
     schedule,
   };
 }