diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 9b60b41682..49ee712c6e 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -96,6 +96,13 @@ catalog: topic: include: ['backstage-include'] # optional array of strings exclude: ['experiments'] # optional array of strings + validateLocationsExist: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + branch: 'main' # string + repository: '.*' # Regex + validateLocationsExist: true # optional boolean enterpriseProviderId: host: ghe.example.net organization: 'backstage' # string @@ -110,7 +117,8 @@ This provider supports multiple organizations via unique provider IDs. - **`catalogPath`** _(optional)_: Default: `/catalog-info.yaml`. Path where to look for `catalog-info.yaml` files. - You can use wildcards - `*` or `**` - to search the path and/or the filename + You can use wildcards - `*` or `**` - to search the path and/or the filename. + Wildcards cannot be used if the `validateLocationsExist` option is set to `true`. - **filters** _(optional)_: - **branch** _(optional)_: String used to filter results based on the branch name. @@ -131,6 +139,12 @@ This provider supports multiple organizations via unique provider IDs. If you want to add multiple organizations, you need to add one provider config each. - **host** _(optional)_: The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md). +- **validateLocationsExist** _(optional)_: + Whether to validate locations that exist before emitting them. + This option avoids generating locations for catalog info files that do not exist in the source repository. + Defaults to `false`. + Due to limitations in the GitHub API's ability to query for repository objects, this option cannot be used in + conjunction with wildcards in the `catalogPath`. ## GitHub API Rate Limits diff --git a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts index c915e8b136..b69728cc62 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts @@ -153,6 +153,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'master', }, + catalogInfoFile: null, }, { name: 'demo', @@ -162,6 +163,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, ], }); @@ -203,6 +205,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, ], }); @@ -234,6 +237,7 @@ describe('GithubDiscoveryProcessor', () => { repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, + catalogInfoFile: null, }, ], }); @@ -259,6 +263,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'master', }, + catalogInfoFile: null, }, ], }); @@ -293,6 +298,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'techdocs-cli', @@ -302,6 +308,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'techdocs-container', @@ -311,6 +318,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'techdocs-durp', @@ -318,6 +326,7 @@ describe('GithubDiscoveryProcessor', () => { repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, + catalogInfoFile: null, }, ], }); @@ -360,6 +369,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'test', @@ -369,6 +379,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'test-archived', @@ -378,6 +389,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'testxyz', @@ -387,6 +399,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, ], }); diff --git a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts index 97c77bf033..593f5f72f1 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts @@ -121,7 +121,11 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { const startTimestamp = Date.now(); this.logger.info(`Reading GitHub repositories from ${location.target}`); - const { repositories } = await getOrganizationRepositories(client, org); + const { repositories } = await getOrganizationRepositories( + client, + org, + catalogPath, + ); const matching = repositories.filter( r => !r.isArchived && repoSearchPath.test(r.name), );