From 4c30622243054e256712e22c5be61a1415fed0f2 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Sun, 28 Aug 2022 14:43:08 +1200 Subject: [PATCH 1/7] Add support for filtering Github repositories by topic Signed-off-by: Marcus Crane --- .../src/lib/github.test.ts | 8 ++++++++ .../src/lib/github.ts | 20 +++++++++++++++++++ .../GithubDiscoveryProcessor.test.ts | 13 ++++++++++++ .../providers/GitHubEntityProvider.test.ts | 8 ++++++++ .../src/providers/GitHubEntityProvider.ts | 5 +++++ .../GitHubEntityProviderConfig.test.ts | 18 ++++++++++++++++- .../providers/GitHubEntityProviderConfig.ts | 3 +++ 7 files changed, 74 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 40dec0a297..e2303c16e6 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -207,6 +207,9 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + repositoryTopics: { + nodes: [{ topic: { name: 'blah' } }], + }, defaultBranchRef: { name: 'main', }, @@ -215,6 +218,7 @@ describe('github', () => { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, @@ -233,6 +237,9 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + repositoryTopics: { + nodes: [{ topic: { name: 'blah' } }], + }, defaultBranchRef: { name: 'main', }, @@ -241,6 +248,7 @@ describe('github', () => { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 888df380f8..6a944af330 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -61,11 +61,22 @@ export type Repository = { name: string; url: string; isArchived: boolean; + repositoryTopics: RepositoryTopics; defaultBranchRef: { name: string; } | null; }; +type RepositoryTopics = { + nodes: TopicNodes[]; +}; + +type TopicNodes = { + topic: { + name: string; + }; +}; + export type Connection = { pageInfo: PageInfo; nodes: T[]; @@ -265,6 +276,15 @@ export async function getOrganizationRepositories( name url isArchived + repositoryTopics(first: 100) { + nodes { + ... on RepositoryTopic { + topic { + name + } + } + } + } defaultBranchRef { name } diff --git a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts index 7ce6f3f9cf..c915e8b136 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts @@ -148,6 +148,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/backstage', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'master', @@ -156,6 +157,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'demo', url: 'https://github.com/backstage/demo', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -196,6 +198,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/tech-docs', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -228,6 +231,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/tech-docs', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, }, @@ -250,6 +254,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/backstage', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'master', @@ -283,6 +288,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/backstage', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -291,6 +297,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'techdocs-cli', url: 'https://github.com/backstage/techdocs-cli', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -299,6 +306,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'techdocs-container', url: 'https://github.com/backstage/techdocs-container', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -307,6 +315,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'techdocs-durp', url: 'https://github.com/backstage/techdocs-durp', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, }, @@ -347,6 +356,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'abstest', url: 'https://github.com/backstage/abctest', isArchived: false, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, @@ -355,6 +365,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'test', url: 'https://github.com/backstage/test', isArchived: false, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, @@ -362,6 +373,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'test-archived', url: 'https://github.com/backstage/test', + repositoryTopics: { nodes: [] }, isArchived: true, defaultBranchRef: { name: 'main', @@ -370,6 +382,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'testxyz', url: 'https://github.com/backstage/testxyz', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 5cc464d345..9d66f66bec 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -116,6 +116,7 @@ describe('GitHubEntityProvider', () => { filters: { branch: 'main', repository: 'test-.*', + topic: 'blah', }, }, }, @@ -143,6 +144,13 @@ describe('GitHubEntityProvider', () => { { name: 'test-repo', url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [ + { + topic: { name: 'blah' }, + }, + ], + }, isArchived: false, defaultBranchRef: { name: 'main', diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index eb5b9d67e2..3e54ed9285 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -183,11 +183,16 @@ export class GitHubEntityProvider implements EntityProvider { private matchesFilters(repositories: Repository[]) { const repositoryFilter = this.config.filters?.repository; + const topicFilter = this.config.filters?.topic; const matchingRepositories = repositories.filter(r => { + const topics: string[] = r.repositoryTopics.nodes.map( + node => node.topic.name, + ); return ( !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && + (!topicFilter || topics.includes(topicFilter)) && r.defaultBranchRef?.name ); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts index ba72dbf950..1a41ddfa5c 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -68,13 +68,19 @@ describe('readProviderConfigs', () => { branch: 'branch-name', }, }, + providerWithTopicFilter: { + organization: 'test-org5', + filters: { + topic: 'backstage-exclude', + }, + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(4); + expect(providerConfigs).toHaveLength(5); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -111,5 +117,15 @@ describe('readProviderConfigs', () => { branch: 'branch-name', }, }); + expect(providerConfigs[4]).toEqual({ + id: 'providerWithTopicFilter', + organization: 'test-org5', + catalogPath: '/catalog-info.yaml', + filters: { + repository: undefined, + branch: undefined, + topic: 'backstage-exclude', + }, + }); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts index b5929c5300..1f5e3d193e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -26,6 +26,7 @@ export type GitHubEntityProviderConfig = { filters?: { repository?: RegExp; branch?: string; + topic?: string; }; }; @@ -58,6 +59,7 @@ function readProviderConfig( config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH; const repositoryPattern = config.getOptionalString('filters.repository'); const branchPattern = config.getOptionalString('filters.branch'); + const topicPattern = config.getOptionalString('filters.topic'); return { id, @@ -68,6 +70,7 @@ function readProviderConfig( ? compileRegExp(repositoryPattern) : undefined, branch: branchPattern || undefined, + topic: topicPattern || undefined, }, }; } From 8de87de128ed3909faed4ab2b026006e1eafce90 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Sun, 28 Aug 2022 15:07:27 +1200 Subject: [PATCH 2/7] Add ability to use topic for inclusion or exclusion Signed-off-by: Marcus Crane --- .../providers/GitHubEntityProvider.test.ts | 155 +++++++++++++++++- .../src/providers/GitHubEntityProvider.ts | 4 +- .../GitHubEntityProviderConfig.test.ts | 29 +++- .../providers/GitHubEntityProviderConfig.ts | 6 + 4 files changed, 191 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 9d66f66bec..ef3a4dbbdd 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -105,7 +105,91 @@ describe('GitHubEntityProvider', () => { ); }); - it('apply full update on scheduled execution', async () => { + it('apply full update on scheduled execution with basic filters', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { nodes: [] }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); + + it('apply full update on scheduled execution with topic exclusion', async () => { const config = new ConfigReader({ catalog: { providers: { @@ -166,6 +250,75 @@ describe('GitHubEntityProvider', () => { expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); await (taskDef.fn as () => Promise)(); + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: [], + }); + }); + + it('apply full update on scheduled execution with topic inclusion', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + topic: 'backstage-include', + topicIncludesIfMatch: true, + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-include' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`; const expectedEntities = [ { diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index 3e54ed9285..9bb0c990df 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -184,6 +184,7 @@ export class GitHubEntityProvider implements EntityProvider { private matchesFilters(repositories: Repository[]) { const repositoryFilter = this.config.filters?.repository; const topicFilter = this.config.filters?.topic; + const topicIncludesIfMatch = this.config.filters?.topicIncludesIfMatch; const matchingRepositories = repositories.filter(r => { const topics: string[] = r.repositoryTopics.nodes.map( @@ -192,7 +193,8 @@ export class GitHubEntityProvider implements EntityProvider { return ( !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && - (!topicFilter || topics.includes(topicFilter)) && + (!topicFilter || + (topics.includes(topicFilter) && topicIncludesIfMatch)) && r.defaultBranchRef?.name ); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts index 1a41ddfa5c..2648129c6f 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -74,13 +74,20 @@ describe('readProviderConfigs', () => { topic: 'backstage-exclude', }, }, + providerWithTopicFilterAndInclusion: { + organization: 'test-org6', + filters: { + topic: 'backstage-include', + topicIncludesIfMatch: false, + }, + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(5); + expect(providerConfigs).toHaveLength(6); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -88,6 +95,8 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, + topic: undefined, + topicIncludesIfMatch: false, }, }); expect(providerConfigs[1]).toEqual({ @@ -97,6 +106,8 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, + topic: undefined, + topicIncludesIfMatch: false, }, }); expect(providerConfigs[2]).toEqual({ @@ -106,6 +117,8 @@ describe('readProviderConfigs', () => { filters: { repository: /^repository.*filter$/, // repo branch: undefined, // branch + topic: undefined, + topicIncludesIfMatch: false, }, }); expect(providerConfigs[3]).toEqual({ @@ -115,6 +128,8 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: 'branch-name', + topic: undefined, + topicIncludesIfMatch: false, }, }); expect(providerConfigs[4]).toEqual({ @@ -125,6 +140,18 @@ describe('readProviderConfigs', () => { repository: undefined, branch: undefined, topic: 'backstage-exclude', + topicIncludesIfMatch: false, + }, + }); + expect(providerConfigs[5]).toEqual({ + id: 'providerWithTopicFilterAndInclusion', + organization: 'test-org6', + catalogPath: '/catalog-info.yaml', + filters: { + repository: undefined, + branch: undefined, + topic: 'backstage-include', + topicIncludesIfMatch: false, }, }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts index 1f5e3d193e..104f12c17c 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; const DEFAULT_PROVIDER_ID = 'default'; +const DEFAULT_TOPIC_INCLUSION = false; export type GitHubEntityProviderConfig = { id: string; @@ -27,6 +28,7 @@ export type GitHubEntityProviderConfig = { repository?: RegExp; branch?: string; topic?: string; + topicIncludesIfMatch: boolean; }; }; @@ -60,6 +62,9 @@ function readProviderConfig( const repositoryPattern = config.getOptionalString('filters.repository'); const branchPattern = config.getOptionalString('filters.branch'); const topicPattern = config.getOptionalString('filters.topic'); + const topicIncludesIfMatch = + config.getOptionalBoolean('filters.topicIncludesIfMatch') ?? + DEFAULT_TOPIC_INCLUSION; return { id, @@ -71,6 +76,7 @@ function readProviderConfig( : undefined, branch: branchPattern || undefined, topic: topicPattern || undefined, + topicIncludesIfMatch, }, }; } From 3a62594a11173bd790b0ef02cd05981cc532ed67 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Sun, 28 Aug 2022 15:35:38 +1200 Subject: [PATCH 3/7] Add changeset Signed-off-by: Marcus Crane --- .changeset/shiny-ties-leave.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/shiny-ties-leave.md diff --git a/.changeset/shiny-ties-leave.md b/.changeset/shiny-ties-leave.md new file mode 100644 index 0000000000..8e146e9af1 --- /dev/null +++ b/.changeset/shiny-ties-leave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Add support for including (or excluding) Github repositories by topic From dc9ea910b2eeab1306af626fe011c78abebe0335 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Sun, 28 Aug 2022 16:02:47 +1200 Subject: [PATCH 4/7] Add discovery documentation Signed-off-by: Marcus Crane --- docs/integrations/github/discovery.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 339a1e2ca3..93e7bd69ae 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -80,6 +80,21 @@ catalog: filters: # optional filters branch: 'develop' # optional string repository: '.*' # optional Regex + topicProviderId: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + branch: 'main' # string + repository: '.*' # Regex + topic: 'backstage-exclude' # optional string + topicProviderInclusionId: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + branch: 'main' # string + repository: '.*' # Regex + topic: 'backstage-include' # optional string + topicIncludesIfMatch: true # optional boolean (default: false) ``` This provider supports multiple organizations via unique provider IDs. @@ -96,6 +111,12 @@ This provider supports multiple organizations via unique provider IDs. String used to filter results based on the branch name. - **repository** _(optional)_: Regular expression used to filter results based on the repository name. + - **topic** _(optional)_: + String used to filter results based on repository topics. By default, repositories that match the topic filter will be **ignored**. See below for the inverse. + This is useful for excluding excess locations from being registered such as hackathon experiments, practical tests, non-production tooling and so on. + - **topicIncludesIfMatch** _(optional)_: + Default: `false` + If set to `true` and a topic filter is set, **ONLY** repositories matching the topic filter will have a location registered. - **organization**: Name of your organization account/workspace. If you want to add multiple organizations, you need to add one provider config each. From fd57abb2007c80c72890c42ea43790c499e296d0 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Sun, 28 Aug 2022 16:08:46 +1200 Subject: [PATCH 5/7] Remove the term hackathon Signed-off-by: Marcus Crane --- docs/integrations/github/discovery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 93e7bd69ae..874df46dc3 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -113,7 +113,7 @@ This provider supports multiple organizations via unique provider IDs. Regular expression used to filter results based on the repository name. - **topic** _(optional)_: String used to filter results based on repository topics. By default, repositories that match the topic filter will be **ignored**. See below for the inverse. - This is useful for excluding excess locations from being registered such as hackathon experiments, practical tests, non-production tooling and so on. + This is useful for excluding excess locations from being registered such as experiments, practical tests, non-production tooling and so on. - **topicIncludesIfMatch** _(optional)_: Default: `false` If set to `true` and a topic filter is set, **ONLY** repositories matching the topic filter will have a location registered. From e6d5534126fa64b42992aa11839ad1a182165e37 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Mon, 29 Aug 2022 20:32:30 +1200 Subject: [PATCH 6/7] Switch to a clearer enum setup and default to ex clude Signed-off-by: Marcus Crane --- docs/integrations/github/discovery.md | 17 ++++--- .../providers/GitHubEntityProvider.test.ts | 10 ++-- .../src/providers/GitHubEntityProvider.ts | 21 +++++++-- .../GitHubEntityProviderConfig.test.ts | 46 +++++++++++++------ .../providers/GitHubEntityProviderConfig.ts | 32 +++++++++---- 5 files changed, 88 insertions(+), 38 deletions(-) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 874df46dc3..b7e9fd031f 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -93,8 +93,9 @@ catalog: filters: branch: 'main' # string repository: '.*' # Regex - topic: 'backstage-include' # optional string - topicIncludesIfMatch: true # optional boolean (default: false) + topic: + name: 'backstage-include' # optional string + type: 'INCLUDES' # optional enum (INCLUDES or EXCLUDES) ``` This provider supports multiple organizations via unique provider IDs. @@ -112,11 +113,13 @@ This provider supports multiple organizations via unique provider IDs. - **repository** _(optional)_: Regular expression used to filter results based on the repository name. - **topic** _(optional)_: - String used to filter results based on repository topics. By default, repositories that match the topic filter will be **ignored**. See below for the inverse. - This is useful for excluding excess locations from being registered such as experiments, practical tests, non-production tooling and so on. - - **topicIncludesIfMatch** _(optional)_: - Default: `false` - If set to `true` and a topic filter is set, **ONLY** repositories matching the topic filter will have a location registered. + - **name** _(optional)_: + String used to filter results based on repository topics. If configured, only repositories that match the topic filter will be registered. See below for the reverse. + This is useful for excluding excess locations from being registered such as experiments, practical tests, non-production tooling and so on. + - **type** _(optional)_: + Default: `INCLUDES` + When set to `INCLUDES`, locations matching the topic filter will be registered and other repositories will be ignored + When set to `EXCLUDES`, locations matching the topic filter will not be registered and other repositories will be registered - **organization**: Name of your organization account/workspace. If you want to add multiple organizations, you need to add one provider config each. diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index ef3a4dbbdd..3156bc3a3d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -200,7 +200,9 @@ describe('GitHubEntityProvider', () => { filters: { branch: 'main', repository: 'test-.*', - topic: 'blah', + topic: { + name: 'blah', + }, }, }, }, @@ -268,8 +270,10 @@ describe('GitHubEntityProvider', () => { filters: { branch: 'main', repository: 'test-.*', - topic: 'backstage-include', - topicIncludesIfMatch: true, + topic: { + name: 'backstage-include', + type: 'INCLUDES', + }, }, }, }, diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index 9bb0c990df..e77e61cf7e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -36,6 +36,7 @@ import { Logger } from 'winston'; import { readProviderConfigs, GitHubEntityProviderConfig, + TopicTypes, } from './GitHubEntityProviderConfig'; import { getOrganizationRepositories, Repository } from '../lib/github'; @@ -184,17 +185,29 @@ export class GitHubEntityProvider implements EntityProvider { private matchesFilters(repositories: Repository[]) { const repositoryFilter = this.config.filters?.repository; const topicFilter = this.config.filters?.topic; - const topicIncludesIfMatch = this.config.filters?.topicIncludesIfMatch; const matchingRepositories = repositories.filter(r => { - const topics: string[] = r.repositoryTopics.nodes.map( + const repoTopics: string[] = r.repositoryTopics.nodes.map( node => node.topic.name, ); + const satisfiesTopicFilter = ( + topics: string[], + filter: typeof topicFilter, + ) => { + if (!filter || !filter.name) return true; + if (filter.type === TopicTypes.Includes && topics.includes(filter.name)) + return true; + if ( + filter.type === TopicTypes.Excludes && + !topics.includes(filter.name) + ) + return true; + return false; + }; return ( !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && - (!topicFilter || - (topics.includes(topicFilter) && topicIncludesIfMatch)) && + satisfiesTopicFilter(repoTopics, topicFilter) && r.defaultBranchRef?.name ); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts index 2648129c6f..6db89f62d8 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -71,14 +71,18 @@ describe('readProviderConfigs', () => { providerWithTopicFilter: { organization: 'test-org5', filters: { - topic: 'backstage-exclude', + topic: { + name: 'backstage-exclude', + }, }, }, providerWithTopicFilterAndInclusion: { organization: 'test-org6', filters: { - topic: 'backstage-include', - topicIncludesIfMatch: false, + topic: { + name: 'backstage-include', + type: 'INCLUDES', + }, }, }, }, @@ -95,8 +99,10 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, - topic: undefined, - topicIncludesIfMatch: false, + topic: { + name: undefined, + type: 'EXCLUDES', + }, }, }); expect(providerConfigs[1]).toEqual({ @@ -106,8 +112,10 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, - topic: undefined, - topicIncludesIfMatch: false, + topic: { + name: undefined, + type: 'EXCLUDES', + }, }, }); expect(providerConfigs[2]).toEqual({ @@ -117,8 +125,10 @@ describe('readProviderConfigs', () => { filters: { repository: /^repository.*filter$/, // repo branch: undefined, // branch - topic: undefined, - topicIncludesIfMatch: false, + topic: { + name: undefined, + type: 'EXCLUDES', + }, }, }); expect(providerConfigs[3]).toEqual({ @@ -128,8 +138,10 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: 'branch-name', - topic: undefined, - topicIncludesIfMatch: false, + topic: { + name: undefined, + type: 'EXCLUDES', + }, }, }); expect(providerConfigs[4]).toEqual({ @@ -139,8 +151,10 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, - topic: 'backstage-exclude', - topicIncludesIfMatch: false, + topic: { + name: 'backstage-exclude', + type: 'EXCLUDES', + }, }, }); expect(providerConfigs[5]).toEqual({ @@ -150,8 +164,10 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, - topic: 'backstage-include', - topicIncludesIfMatch: false, + topic: { + name: 'backstage-include', + type: 'INCLUDES', + }, }, }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts index 104f12c17c..609cdf163d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -16,9 +16,17 @@ import { Config } from '@backstage/config'; +export enum TopicTypes { + Includes = 'INCLUDES', + Excludes = 'EXCLUDES', +} + +const isValidTopicType = (topicType: string) => + (Object.values(TopicTypes) as string[]).includes(topicType); + const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; const DEFAULT_PROVIDER_ID = 'default'; -const DEFAULT_TOPIC_INCLUSION = false; +const DEFAULT_TOPIC_FILTER_TYPE: TopicTypes = TopicTypes.Excludes; export type GitHubEntityProviderConfig = { id: string; @@ -27,8 +35,10 @@ export type GitHubEntityProviderConfig = { filters?: { repository?: RegExp; branch?: string; - topic?: string; - topicIncludesIfMatch: boolean; + topic?: { + name?: string; + type: TopicTypes; + }; }; }; @@ -61,10 +71,9 @@ function readProviderConfig( config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH; const repositoryPattern = config.getOptionalString('filters.repository'); const branchPattern = config.getOptionalString('filters.branch'); - const topicPattern = config.getOptionalString('filters.topic'); - const topicIncludesIfMatch = - config.getOptionalBoolean('filters.topicIncludesIfMatch') ?? - DEFAULT_TOPIC_INCLUSION; + const topicFilterConfig = config.getOptionalConfig('filters.topic'); + const topicFilterName = topicFilterConfig?.getOptionalString('name'); + const topicFilterType = topicFilterConfig?.getOptionalString('type'); return { id, @@ -75,8 +84,13 @@ function readProviderConfig( ? compileRegExp(repositoryPattern) : undefined, branch: branchPattern || undefined, - topic: topicPattern || undefined, - topicIncludesIfMatch, + topic: { + name: topicFilterName || undefined, + type: + topicFilterType && isValidTopicType(topicFilterType) + ? (topicFilterType as TopicTypes) + : DEFAULT_TOPIC_FILTER_TYPE, + }, }, }; } From 4da5a2fc40bf00d527f561ac68be7485513d1c43 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Sat, 3 Sep 2022 17:01:40 +1200 Subject: [PATCH 7/7] Updated documentation to reflect new topic filter API Signed-off-by: Marcus Crane --- docs/integrations/github/discovery.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index b7e9fd031f..6966753d68 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -87,15 +87,15 @@ catalog: branch: 'main' # string repository: '.*' # Regex topic: 'backstage-exclude' # optional string - topicProviderInclusionId: + topicFilterProviderId: organization: 'backstage' # string catalogPath: '/catalog-info.yaml' # string filters: branch: 'main' # string repository: '.*' # Regex topic: - name: 'backstage-include' # optional string - type: 'INCLUDES' # optional enum (INCLUDES or EXCLUDES) + include: ['backstage-include'] # optional array of strings + exclude: ['experiments'] # optional array of strings ``` This provider supports multiple organizations via unique provider IDs. @@ -113,13 +113,15 @@ This provider supports multiple organizations via unique provider IDs. - **repository** _(optional)_: Regular expression used to filter results based on the repository name. - **topic** _(optional)_: - - **name** _(optional)_: - String used to filter results based on repository topics. If configured, only repositories that match the topic filter will be registered. See below for the reverse. - This is useful for excluding excess locations from being registered such as experiments, practical tests, non-production tooling and so on. - - **type** _(optional)_: - Default: `INCLUDES` - When set to `INCLUDES`, locations matching the topic filter will be registered and other repositories will be ignored - When set to `EXCLUDES`, locations matching the topic filter will not be registered and other repositories will be registered + Both of the filters below may be used at the same time but the exclusion filter has the highest priority. + In the example above, a repository with the `backstage-include` topic would still be excluded + if it were also carrying the `experiments` topic. + - **include** _(optional)_: + An array of strings used to filter in results based on their associated Github topics. + If configured, only repositories with one (or more) topic(s) present in the inclusion filter will be ingested + - **exclude** _(optional)_: + An array of strings used to filter out results based on their associated Github topics. + If configured, all repositories _except_ those with one (or more) topics(s) present in the exclusion filter will be ingested. - **organization**: Name of your organization account/workspace. If you want to add multiple organizations, you need to add one provider config each.