From 8de87de128ed3909faed4ab2b026006e1eafce90 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Sun, 28 Aug 2022 15:07:27 +1200 Subject: [PATCH] Add ability to use topic for inclusion or exclusion Signed-off-by: Marcus Crane --- .../providers/GitHubEntityProvider.test.ts | 155 +++++++++++++++++- .../src/providers/GitHubEntityProvider.ts | 4 +- .../GitHubEntityProviderConfig.test.ts | 29 +++- .../providers/GitHubEntityProviderConfig.ts | 6 + 4 files changed, 191 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 9d66f66bec..ef3a4dbbdd 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -105,7 +105,91 @@ describe('GitHubEntityProvider', () => { ); }); - it('apply full update on scheduled execution', async () => { + it('apply full update on scheduled execution with basic filters', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { nodes: [] }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); + + it('apply full update on scheduled execution with topic exclusion', async () => { const config = new ConfigReader({ catalog: { providers: { @@ -166,6 +250,75 @@ describe('GitHubEntityProvider', () => { expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); await (taskDef.fn as () => Promise)(); + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: [], + }); + }); + + it('apply full update on scheduled execution with topic inclusion', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + topic: 'backstage-include', + topicIncludesIfMatch: true, + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-include' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`; const expectedEntities = [ { diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index 3e54ed9285..9bb0c990df 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -184,6 +184,7 @@ export class GitHubEntityProvider implements EntityProvider { private matchesFilters(repositories: Repository[]) { const repositoryFilter = this.config.filters?.repository; const topicFilter = this.config.filters?.topic; + const topicIncludesIfMatch = this.config.filters?.topicIncludesIfMatch; const matchingRepositories = repositories.filter(r => { const topics: string[] = r.repositoryTopics.nodes.map( @@ -192,7 +193,8 @@ export class GitHubEntityProvider implements EntityProvider { return ( !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && - (!topicFilter || topics.includes(topicFilter)) && + (!topicFilter || + (topics.includes(topicFilter) && topicIncludesIfMatch)) && r.defaultBranchRef?.name ); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts index 1a41ddfa5c..2648129c6f 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -74,13 +74,20 @@ describe('readProviderConfigs', () => { topic: 'backstage-exclude', }, }, + providerWithTopicFilterAndInclusion: { + organization: 'test-org6', + filters: { + topic: 'backstage-include', + topicIncludesIfMatch: false, + }, + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(5); + expect(providerConfigs).toHaveLength(6); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -88,6 +95,8 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, + topic: undefined, + topicIncludesIfMatch: false, }, }); expect(providerConfigs[1]).toEqual({ @@ -97,6 +106,8 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: undefined, + topic: undefined, + topicIncludesIfMatch: false, }, }); expect(providerConfigs[2]).toEqual({ @@ -106,6 +117,8 @@ describe('readProviderConfigs', () => { filters: { repository: /^repository.*filter$/, // repo branch: undefined, // branch + topic: undefined, + topicIncludesIfMatch: false, }, }); expect(providerConfigs[3]).toEqual({ @@ -115,6 +128,8 @@ describe('readProviderConfigs', () => { filters: { repository: undefined, branch: 'branch-name', + topic: undefined, + topicIncludesIfMatch: false, }, }); expect(providerConfigs[4]).toEqual({ @@ -125,6 +140,18 @@ describe('readProviderConfigs', () => { repository: undefined, branch: undefined, topic: 'backstage-exclude', + topicIncludesIfMatch: false, + }, + }); + expect(providerConfigs[5]).toEqual({ + id: 'providerWithTopicFilterAndInclusion', + organization: 'test-org6', + catalogPath: '/catalog-info.yaml', + filters: { + repository: undefined, + branch: undefined, + topic: 'backstage-include', + topicIncludesIfMatch: false, }, }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts index 1f5e3d193e..104f12c17c 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; const DEFAULT_PROVIDER_ID = 'default'; +const DEFAULT_TOPIC_INCLUSION = false; export type GitHubEntityProviderConfig = { id: string; @@ -27,6 +28,7 @@ export type GitHubEntityProviderConfig = { repository?: RegExp; branch?: string; topic?: string; + topicIncludesIfMatch: boolean; }; }; @@ -60,6 +62,9 @@ function readProviderConfig( const repositoryPattern = config.getOptionalString('filters.repository'); const branchPattern = config.getOptionalString('filters.branch'); const topicPattern = config.getOptionalString('filters.topic'); + const topicIncludesIfMatch = + config.getOptionalBoolean('filters.topicIncludesIfMatch') ?? + DEFAULT_TOPIC_INCLUSION; return { id, @@ -71,6 +76,7 @@ function readProviderConfig( : undefined, branch: branchPattern || undefined, topic: topicPattern || undefined, + topicIncludesIfMatch, }, }; }