From 4c30622243054e256712e22c5be61a1415fed0f2 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Sun, 28 Aug 2022 14:43:08 +1200 Subject: [PATCH] Add support for filtering Github repositories by topic Signed-off-by: Marcus Crane --- .../src/lib/github.test.ts | 8 ++++++++ .../src/lib/github.ts | 20 +++++++++++++++++++ .../GithubDiscoveryProcessor.test.ts | 13 ++++++++++++ .../providers/GitHubEntityProvider.test.ts | 8 ++++++++ .../src/providers/GitHubEntityProvider.ts | 5 +++++ .../GitHubEntityProviderConfig.test.ts | 18 ++++++++++++++++- .../providers/GitHubEntityProviderConfig.ts | 3 +++ 7 files changed, 74 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 40dec0a297..e2303c16e6 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -207,6 +207,9 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + repositoryTopics: { + nodes: [{ topic: { name: 'blah' } }], + }, defaultBranchRef: { name: 'main', }, @@ -215,6 +218,7 @@ describe('github', () => { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, @@ -233,6 +237,9 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + repositoryTopics: { + nodes: [{ topic: { name: 'blah' } }], + }, defaultBranchRef: { name: 'main', }, @@ -241,6 +248,7 @@ describe('github', () => { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 888df380f8..6a944af330 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -61,11 +61,22 @@ export type Repository = { name: string; url: string; isArchived: boolean; + repositoryTopics: RepositoryTopics; defaultBranchRef: { name: string; } | null; }; +type RepositoryTopics = { + nodes: TopicNodes[]; +}; + +type TopicNodes = { + topic: { + name: string; + }; +}; + export type Connection = { pageInfo: PageInfo; nodes: T[]; @@ -265,6 +276,15 @@ export async function getOrganizationRepositories( name url isArchived + repositoryTopics(first: 100) { + nodes { + ... on RepositoryTopic { + topic { + name + } + } + } + } defaultBranchRef { name } diff --git a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts index 7ce6f3f9cf..c915e8b136 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts @@ -148,6 +148,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/backstage', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'master', @@ -156,6 +157,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'demo', url: 'https://github.com/backstage/demo', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -196,6 +198,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/tech-docs', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -228,6 +231,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/tech-docs', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, }, @@ -250,6 +254,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/backstage', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'master', @@ -283,6 +288,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'backstage', url: 'https://github.com/backstage/backstage', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -291,6 +297,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'techdocs-cli', url: 'https://github.com/backstage/techdocs-cli', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -299,6 +306,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'techdocs-container', url: 'https://github.com/backstage/techdocs-container', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', @@ -307,6 +315,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'techdocs-durp', url: 'https://github.com/backstage/techdocs-durp', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, }, @@ -347,6 +356,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'abstest', url: 'https://github.com/backstage/abctest', isArchived: false, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, @@ -355,6 +365,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'test', url: 'https://github.com/backstage/test', isArchived: false, + repositoryTopics: { nodes: [] }, defaultBranchRef: { name: 'main', }, @@ -362,6 +373,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'test-archived', url: 'https://github.com/backstage/test', + repositoryTopics: { nodes: [] }, isArchived: true, defaultBranchRef: { name: 'main', @@ -370,6 +382,7 @@ describe('GithubDiscoveryProcessor', () => { { name: 'testxyz', url: 'https://github.com/backstage/testxyz', + repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: { name: 'main', diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 5cc464d345..9d66f66bec 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -116,6 +116,7 @@ describe('GitHubEntityProvider', () => { filters: { branch: 'main', repository: 'test-.*', + topic: 'blah', }, }, }, @@ -143,6 +144,13 @@ describe('GitHubEntityProvider', () => { { name: 'test-repo', url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [ + { + topic: { name: 'blah' }, + }, + ], + }, isArchived: false, defaultBranchRef: { name: 'main', diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index eb5b9d67e2..3e54ed9285 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -183,11 +183,16 @@ export class GitHubEntityProvider implements EntityProvider { private matchesFilters(repositories: Repository[]) { const repositoryFilter = this.config.filters?.repository; + const topicFilter = this.config.filters?.topic; const matchingRepositories = repositories.filter(r => { + const topics: string[] = r.repositoryTopics.nodes.map( + node => node.topic.name, + ); return ( !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && + (!topicFilter || topics.includes(topicFilter)) && r.defaultBranchRef?.name ); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts index ba72dbf950..1a41ddfa5c 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -68,13 +68,19 @@ describe('readProviderConfigs', () => { branch: 'branch-name', }, }, + providerWithTopicFilter: { + organization: 'test-org5', + filters: { + topic: 'backstage-exclude', + }, + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(4); + expect(providerConfigs).toHaveLength(5); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -111,5 +117,15 @@ describe('readProviderConfigs', () => { branch: 'branch-name', }, }); + expect(providerConfigs[4]).toEqual({ + id: 'providerWithTopicFilter', + organization: 'test-org5', + catalogPath: '/catalog-info.yaml', + filters: { + repository: undefined, + branch: undefined, + topic: 'backstage-exclude', + }, + }); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts index b5929c5300..1f5e3d193e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -26,6 +26,7 @@ export type GitHubEntityProviderConfig = { filters?: { repository?: RegExp; branch?: string; + topic?: string; }; }; @@ -58,6 +59,7 @@ function readProviderConfig( config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH; const repositoryPattern = config.getOptionalString('filters.repository'); const branchPattern = config.getOptionalString('filters.branch'); + const topicPattern = config.getOptionalString('filters.topic'); return { id, @@ -68,6 +70,7 @@ function readProviderConfig( ? compileRegExp(repositoryPattern) : undefined, branch: branchPattern || undefined, + topic: topicPattern || undefined, }, }; }