diff --git a/.changeset/shiny-ties-leave.md b/.changeset/shiny-ties-leave.md new file mode 100644 index 0000000000..8e146e9af1 --- /dev/null +++ b/.changeset/shiny-ties-leave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Add support for including (or excluding) Github repositories by topic diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 339a1e2ca3..6966753d68 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -80,6 +80,22 @@ catalog: filters: # optional filters branch: 'develop' # optional string repository: '.*' # optional Regex + topicProviderId: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + branch: 'main' # string + repository: '.*' # Regex + topic: 'backstage-exclude' # optional string + topicFilterProviderId: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + branch: 'main' # string + repository: '.*' # Regex + topic: + include: ['backstage-include'] # optional array of strings + exclude: ['experiments'] # optional array of strings ``` This provider supports multiple organizations via unique provider IDs. @@ -96,6 +112,16 @@ This provider supports multiple organizations via unique provider IDs. String used to filter results based on the branch name. - **repository** _(optional)_: Regular expression used to filter results based on the repository name. + - **topic** _(optional)_: + Both of the filters below may be used at the same time but the exclusion filter has the highest priority. + In the example above, a repository with the `backstage-include` topic would still be excluded + if it were also carrying the `experiments` topic. + - **include** _(optional)_: + An array of strings used to filter in results based on their associated Github topics. + If configured, only repositories with one (or more) topic(s) present in the inclusion filter will be ingested + - **exclude** _(optional)_: + An array of strings used to filter out results based on their associated Github topics. + If configured, all repositories _except_ those with one (or more) topics(s) present in the exclusion filter will be ingested. - **organization**: Name of your organization account/workspace. If you want to add multiple organizations, you need to add one provider config each. diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 40dec0a297..e2303c16e6 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -207,6 +207,9 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + repositoryTopics: { + nodes: [{ topic: { name: 'blah' } }], + }, defaultBranchRef: { name: 'main', }, @@ -215,6 +218,7 @@ describe('github', () => { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, @@ -233,6 +237,9 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + repositoryTopics: { + nodes: [{ topic: { name: 'blah' } }], + }, defaultBranchRef: { name: 'main', }, @@ -241,6 +248,7 @@ describe('github', () => { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 888df380f8..6a944af330 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -61,11 +61,22 @@ export type Repository = { name: string; url: string; isArchived: boolean; + repositoryTopics: RepositoryTopics; defaultBranchRef: { name: string; } | null; }; +type RepositoryTopics = { + nodes: TopicNodes[]; +}; + +type TopicNodes = { + topic: { + name: string; + }; +}; + export type Connection = { pageInfo: PageInfo; nodes: T[]; @@ -265,6 +276,15 @@ export async function getOrganizationRepositories( name url isArchived + repositoryTopics(first: 100) { + nodes { + ... on RepositoryTopic { + topic { + name + } + } + } + } defaultBranchRef { name } diff --git a/plugins/catalog-backend-module-github/src/lib/util.test.ts b/plugins/catalog-backend-module-github/src/lib/util.test.ts index c73c11bd2d..c530d2bcc9 100644 --- a/plugins/catalog-backend-module-github/src/lib/util.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/util.test.ts @@ -14,7 +14,8 @@ * limitations under the License. */ -import { parseGitHubOrgUrl } from './util'; +import { GithubTopicFilters } from '../providers/GitHubEntityProviderConfig'; +import { parseGitHubOrgUrl, satisfiesTopicFilter } from './util'; describe('parseGitHubOrgUrl', () => { it('only supports clean org urls, and decodes them', () => { @@ -28,3 +29,61 @@ describe('parseGitHubOrgUrl', () => { }); }); }); + +describe('satisfiesTopicFilter', () => { + it('handles cases where the filter will never apply', () => { + expect(satisfiesTopicFilter([], undefined)).toEqual(true); + expect( + satisfiesTopicFilter([], { include: undefined, exclude: undefined }), + ).toEqual(true); + expect(satisfiesTopicFilter([], { include: [], exclude: [] })).toEqual( + true, + ); + }); + + it('handles cases where include filter is configured', () => { + const filter: GithubTopicFilters = { + include: ['backstage-include', 'open-source'], + exclude: undefined, + }; + expect(satisfiesTopicFilter([], filter)).toEqual(false); + expect(satisfiesTopicFilter(['blah', 'something-else'], filter)).toEqual( + false, + ); + expect(satisfiesTopicFilter(['backstage-include'], filter)).toEqual(true); + expect( + satisfiesTopicFilter(['backstage-include', 'open-source'], filter), + ).toEqual(true); + }); + + it('handles cases where exclude filter is configured', () => { + const filter: GithubTopicFilters = { + include: undefined, + exclude: ['backstage-exclude', 'experiment'], + }; + expect(satisfiesTopicFilter([], filter)).toEqual(true); + expect(satisfiesTopicFilter(['blah', 'bleh'], filter)).toEqual(true); + expect( + satisfiesTopicFilter(['backstage-include', 'backstage-exclude'], filter), + ).toEqual(false); + expect( + satisfiesTopicFilter(['experiment', 'backstage-include'], filter), + ).toEqual(false); + }); + + it('handles cases where both exclusion filters are configured', () => { + const filter: GithubTopicFilters = { + include: ['backstage-include', 'blah'], + exclude: ['backstage-exclude', 'bleh'], + }; + expect(satisfiesTopicFilter([], filter)).toEqual(false); + expect(satisfiesTopicFilter(['backstage-include'], filter)).toEqual(true); + expect(satisfiesTopicFilter(['blah', 'nothing'], filter)).toEqual(true); + expect(satisfiesTopicFilter(['backstage-exclude'], filter)).toEqual(false); + expect(satisfiesTopicFilter(['bleh, nothing'], filter)).toEqual(false); + expect(satisfiesTopicFilter(['abc123'], filter)).toEqual(false); + expect( + satisfiesTopicFilter(['backstage-include', 'backstage-exclude'], filter), + ).toEqual(false); + }); +}); diff --git a/plugins/catalog-backend-module-github/src/lib/util.ts b/plugins/catalog-backend-module-github/src/lib/util.ts index a38225ff98..238ed9868a 100644 --- a/plugins/catalog-backend-module-github/src/lib/util.ts +++ b/plugins/catalog-backend-module-github/src/lib/util.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { GithubTopicFilters } from '../providers/GitHubEntityProviderConfig'; + export function parseGitHubOrgUrl(urlString: string): { org: string } { const path = new URL(urlString).pathname.substr(1).split('/'); @@ -24,3 +26,54 @@ export function parseGitHubOrgUrl(urlString: string): { org: string } { throw new Error(`Expected a URL pointing to /`); } + +export function satisfiesTopicFilter( + topics: string[], + topicFilter: GithubTopicFilters | undefined, +): Boolean { + // We don't want to do anything if a filter is not configured (or configured but empty) + if (!topicFilter) return true; + if (!topicFilter.include && !topicFilter.exclude) return true; + if (!topicFilter.include?.length && !topicFilter.exclude?.length) return true; + // If topic.include is in use, a topic MUST be set that matches the inclusion + // filter in order for a repository to be ingested + if (topicFilter.include?.length && !topicFilter.exclude) { + for (const topic of topics) { + if (topicFilter.include.includes(topic)) return true; + } + return false; + } + // If topic.exclude is in use, all topics are included by default + // with anything matching the filter being discarded. It is technically + // possible for the filter to be explicitly empty meaning all repositories + // are ingested although doing so would be pointless. + if (!topicFilter.include && topicFilter.exclude?.length) { + if (!topics.length) return true; + for (const topic of topics) { + if (topicFilter.exclude.includes(topic)) return false; + } + return true; + } + // Now the tricky part is where we have both include and exclude configured. + // In this case, exclude wins out so we take everything matching the initial + // inclusion filter and from that group, we further discard anything matching + // the exclusion filter. If an item were to have a topic set in both filters, + // we expect it to be discarded in the end. The use case here is that is that + // you may want to retrieve all repos with the topic of team-a while excluding + // matching repos that are marked as experiments. + if (topicFilter.include && topicFilter.exclude) { + const matchesInclude = satisfiesTopicFilter(topics, { + include: topicFilter.include, + }); + const matchesExclude = !satisfiesTopicFilter(topics, { + exclude: topicFilter.exclude, + }); + if (matchesExclude) return false; + return matchesInclude; + } + + // If the topic filter is somehow ever in a state that is not covered here, we + // will fail "open" so that Backstage is still usable as if the filter was + // not configured at all. + return true; +} diff --git a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts index 7ce6f3f9cf..c915e8b136 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts @@ -148,6 +148,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/backstage', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'master', @@ -156,6 +157,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'demo', url: 'https://github.com/backstage/demo', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -196,6 +198,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/tech-docs', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -228,6 +231,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/tech-docs', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, }, @@ -250,6 +254,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/backstage', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'master', @@ -283,6 +288,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/backstage', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -291,6 +297,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'techdocs-cli', url: 'https://github.com/backstage/techdocs-cli', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -299,6 +306,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'techdocs-container', url: 'https://github.com/backstage/techdocs-container', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -307,6 +315,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'techdocs-durp', url: 'https://github.com/backstage/techdocs-durp', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, }, @@ -347,6 +356,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'abstest', url: 'https://github.com/backstage/abctest', isArchived: false, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, @@ -355,6 +365,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'test', url: 'https://github.com/backstage/test', isArchived: false, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, @@ -362,6 +373,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'test-archived', url: 'https://github.com/backstage/test', + repositoryTopics: { nodes: [] }, isArchived: true, defaultBranchRef: { name: 'main', @@ -370,6 +382,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'testxyz', url: 'https://github.com/backstage/testxyz', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 1c51969c1d..871718cc05 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -126,7 +126,7 @@ describe('GitHubEntityProvider', () => { ); }); - it('apply full update on scheduled execution', async () => { + it('apply full update on scheduled execution with basic filters', async () => { const config = new ConfigReader({ catalog: { providers: { @@ -164,6 +164,176 @@ describe('GitHubEntityProvider', () => { { name: 'test-repo', url: 'https://github.com/test-org/test-repo', + repositoryTopics: { nodes: [] }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); + + it('apply full update on scheduled execution with topic exclusion', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + topic: { + exclude: ['backstage-exclude'], + }, + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-exclude' }, + }, + { + topic: { name: 'neat-repos' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [], + }); + }); + + it('apply full update on scheduled execution with topic inclusion', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + topic: { + include: ['backstage-include'], + }, + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-include' }, + }, + { + topic: { name: 'fruits' }, + }, + ], + }, isArchived: false, defaultBranchRef: { name: 'main', @@ -209,3 +379,130 @@ describe('GitHubEntityProvider', () => { }); }); }); + +it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + topic: { + exclude: ['backstage-exclude'], + include: ['backstage-include'], + }, + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-include' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + { + name: 'test-repo-2', + url: 'https://github.com/test-org/test-repo-2', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-include' }, + }, + { + topic: { name: 'backstage-exclude' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + { + name: 'test-repo-3', + url: 'https://github.com/test-org/test-repo-3', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-exclude' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); +}); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index b369311560..cf9b2cdd0a 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -38,6 +38,7 @@ import { GitHubEntityProviderConfig, } from './GitHubEntityProviderConfig'; import { getOrganizationRepositories, Repository } from '../lib/github'; +import { satisfiesTopicFilter } from '../lib/util'; /** * Discovers catalog files located in [GitHub](https://github.com). @@ -184,11 +185,16 @@ export class GitHubEntityProvider implements EntityProvider { private matchesFilters(repositories: Repository[]) { const repositoryFilter = this.config.filters?.repository; + const topicFilters = this.config.filters?.topic; const matchingRepositories = repositories.filter(r => { + const repoTopics: string[] = r.repositoryTopics.nodes.map( + node => node.topic.name, + ); return ( !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && + satisfiesTopicFilter(repoTopics, topicFilters) && r.defaultBranchRef?.name ); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts index bc13b7b5d4..66a5463bf5 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -68,6 +68,15 @@ describe('readProviderConfigs', () => { branch: 'branch-name', }, }, + providerWithTopicFilter: { + organization: 'test-org5', + filters: { + topic: { + exclude: ['backstage-exclude'], + include: ['backstage-include'], + }, + }, + }, providerWithHost: { organization: 'test-org1', host: 'ghe.internal.com', @@ -78,7 +87,7 @@ describe('readProviderConfigs', () => { }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(5); + expect(providerConfigs).toHaveLength(6); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -87,6 +96,10 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, + topic: { + include: undefined, + exclude: undefined, + }, }, }); expect(providerConfigs[1]).toEqual({ @@ -97,6 +110,10 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, + topic: { + include: undefined, + exclude: undefined, + }, }, }); expect(providerConfigs[2]).toEqual({ @@ -107,6 +124,10 @@ describe('readProviderConfigs', () => { filters: { repository: /^repository.*filter$/, // repo branch: undefined, // branch + topic: { + include: undefined, + exclude: undefined, + }, }, }); expect(providerConfigs[3]).toEqual({ @@ -117,9 +138,27 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: 'branch-name', + topic: { + include: undefined, + exclude: undefined, + }, }, }); expect(providerConfigs[4]).toEqual({ + id: 'providerWithTopicFilter', + organization: 'test-org5', + catalogPath: '/catalog-info.yaml', + host: 'github.com', + filters: { + repository: undefined, + branch: undefined, + topic: { + include: ['backstage-include'], + exclude: ['backstage-exclude'], + }, + }, + }); + expect(providerConfigs[5]).toEqual({ id: 'providerWithHost', organization: 'test-org1', catalogPath: '/catalog-info.yaml', @@ -127,6 +166,10 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, + topic: { + include: undefined, + exclude: undefined, + }, }, }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts index 18b54b7c4a..d40b206751 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -27,9 +27,15 @@ export type GitHubEntityProviderConfig = { filters?: { repository?: RegExp; branch?: string; + topic?: GithubTopicFilters; }; }; +export type GithubTopicFilters = { + exclude?: string[]; + include?: string[]; +}; + export function readProviderConfigs( config: Config, ): GitHubEntityProviderConfig[] { @@ -60,6 +66,12 @@ function readProviderConfig( const host = config.getOptionalString('host') ?? 'github.com'; const repositoryPattern = config.getOptionalString('filters.repository'); const branchPattern = config.getOptionalString('filters.branch'); + const topicFilterInclude = config?.getOptionalStringArray( + 'filters.topic.include', + ); + const topicFilterExclude = config?.getOptionalStringArray( + 'filters.topic.exclude', + ); return { id, @@ -71,6 +83,10 @@ function readProviderConfig( ? compileRegExp(repositoryPattern) : undefined, branch: branchPattern || undefined, + topic: { + include: topicFilterInclude, + exclude: topicFilterExclude, + }, }, }; }