Add docs for the new validateLocationsExist option

Signed-off-by: Nikolas Skoufis <nskoufis@seek.com.au>

docs/integrations/github/discovery.md

@@ -96,6 +96,13 @@ catalog:
           topic:
             include: ['backstage-include'] # optional array of strings
             exclude: ['experiments'] # optional array of strings
+      validateLocationsExist:
+        organization: 'backstage' # string
+        catalogPath: '/catalog-info.yaml' # string
+        filters:
+          branch: 'main' # string
+          repository: '.*' # Regex
+        validateLocationsExist: true # optional boolean
       enterpriseProviderId:
         host: ghe.example.net
         organization: 'backstage' # string
@@ -110,7 +117,8 @@ This provider supports multiple organizations via unique provider IDs.
 - **`catalogPath`** _(optional)_:
   Default: `/catalog-info.yaml`.
   Path where to look for `catalog-info.yaml` files.
-  You can use wildcards - `*` or `**` - to search the path and/or the filename
+  You can use wildcards - `*` or `**` - to search the path and/or the filename.
+  Wildcards cannot be used if the `validateLocationsExist` option is set to `true`.
 - **filters** _(optional)_:
   - **branch** _(optional)_:
     String used to filter results based on the branch name.
@@ -131,6 +139,12 @@ This provider supports multiple organizations via unique provider IDs.
   If you want to add multiple organizations, you need to add one provider config each.
 - **host** _(optional)_:
   The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md).
+- **validateLocationsExist** _(optional)_:
+  Whether to validate locations that exist before emitting them.
+  This option avoids generating locations for catalog info files that do not exist in the source repository.
+  Defaults to `false`.
+  Due to limitations in the GitHub API's ability to query for repository objects, this option cannot be used in 
+  conjunction with wildcards in the `catalogPath`.
 
 ## GitHub API Rate Limits
 

plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts

@@ -153,6 +153,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'master',
             },
+            catalogInfoFile: null,
           },
           {
             name: 'demo',
@@ -162,6 +163,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
         ],
       });
@@ -203,6 +205,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
         ],
       });
@@ -234,6 +237,7 @@ describe('GithubDiscoveryProcessor', () => {
             repositoryTopics: { nodes: [] },
             isArchived: false,
             defaultBranchRef: null,
+            catalogInfoFile: null,
           },
         ],
       });
@@ -259,6 +263,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'master',
             },
+            catalogInfoFile: null,
           },
         ],
       });
@@ -293,6 +298,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
           {
             name: 'techdocs-cli',
@@ -302,6 +308,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
           {
             name: 'techdocs-container',
@@ -311,6 +318,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
           {
             name: 'techdocs-durp',
@@ -318,6 +326,7 @@ describe('GithubDiscoveryProcessor', () => {
             repositoryTopics: { nodes: [] },
             isArchived: false,
             defaultBranchRef: null,
+            catalogInfoFile: null,
           },
         ],
       });
@@ -360,6 +369,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
           {
             name: 'test',
@@ -369,6 +379,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
           {
             name: 'test-archived',
@@ -378,6 +389,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
           {
             name: 'testxyz',
@@ -387,6 +399,7 @@ describe('GithubDiscoveryProcessor', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
         ],
       });

plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts

@@ -121,7 +121,11 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
     const startTimestamp = Date.now();
     this.logger.info(`Reading GitHub repositories from ${location.target}`);
 
-    const { repositories } = await getOrganizationRepositories(client, org);
+    const { repositories } = await getOrganizationRepositories(
+      client,
+      org,
+      catalogPath,
+    );
     const matching = repositories.filter(
       r => !r.isArchived && repoSearchPath.test(r.name),
     );