diff --git a/.changeset/eleven-balloons-occur.md b/.changeset/eleven-balloons-occur.md new file mode 100644 index 0000000000..c857359aae --- /dev/null +++ b/.changeset/eleven-balloons-occur.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Add visibility filter on GithubEntityProvider diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index fb65173b28..aed22702ac 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -163,6 +163,13 @@ catalog: branch: 'main' # string repository: '.*' # Regex validateLocationsExist: true # optional boolean + visibilityProviderId: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + visibilities: + - public + - internal enterpriseProviderId: host: ghe.example.net organization: 'backstage' # string diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 0345d4ba3e..df4565a174 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -493,6 +493,7 @@ describe('github', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'demo', @@ -508,6 +509,7 @@ describe('github', () => { id: 'acb123', text: 'some yaml', }, + visibility: 'private', }, ], pageInfo: { @@ -531,6 +533,7 @@ describe('github', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'demo', @@ -546,6 +549,7 @@ describe('github', () => { id: 'acb123', text: 'some yaml', }, + visibility: 'private', }, ], }; diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 30e7bd83d4..39501e89bc 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -99,6 +99,7 @@ export type RepositoryResponse = { id: string; text: string; } | null; + visibility: string; }; type RepositoryTopics = { @@ -437,6 +438,7 @@ export async function getOrganizationRepositories( url isArchived isFork + visibility repositoryTopics(first: 100) { nodes { ... on RepositoryTopic { diff --git a/plugins/catalog-backend-module-github/src/lib/util.test.ts b/plugins/catalog-backend-module-github/src/lib/util.test.ts index 0de1d24bbf..656216942d 100644 --- a/plugins/catalog-backend-module-github/src/lib/util.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/util.test.ts @@ -19,6 +19,7 @@ import { parseGithubOrgUrl, satisfiesTopicFilter, satisfiesForkFilter, + satisfiesVisibilityFilter, } from './util'; describe('parseGithubOrgUrl', () => { @@ -109,3 +110,14 @@ describe('satisfiesForkFilter', () => { expect(satisfiesForkFilter(true, false)).toEqual(true); }); }); + +describe('satisfiesVisibilityFilter', () => { + it('satisfies if no visibilities are given', () => { + expect(satisfiesVisibilityFilter([], 'public')).toEqual(true); + }); + + it('handles visibility presence into the visibilities filter', () => { + expect(satisfiesVisibilityFilter(['public'], 'public')).toEqual(true); + expect(satisfiesVisibilityFilter(['private'], 'public')).toEqual(false); + }); +}); diff --git a/plugins/catalog-backend-module-github/src/lib/util.ts b/plugins/catalog-backend-module-github/src/lib/util.ts index 54ee575a6c..9e13814598 100644 --- a/plugins/catalog-backend-module-github/src/lib/util.ts +++ b/plugins/catalog-backend-module-github/src/lib/util.ts @@ -99,3 +99,13 @@ export function splitTeamSlug(slug: string): [string, string] { } return [parts[0], parts[1]]; } + +export function satisfiesVisibilityFilter( + visibilities: string[], + visibility: string, +): Boolean { + if (!visibilities.length) { + return true; + } + return visibilities.indexOf(visibility) > -1; +} diff --git a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts index f008c88dbe..7fbee87161 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts @@ -155,6 +155,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'master', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'demo', @@ -166,6 +167,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, ], }); @@ -209,6 +211,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, ], }); @@ -242,6 +245,7 @@ describe('GithubDiscoveryProcessor', () => { isFork: false, defaultBranchRef: null, catalogInfoFile: null, + visibility: 'public', }, ], }); @@ -269,6 +273,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'master', }, catalogInfoFile: null, + visibility: 'public', }, ], }); @@ -305,6 +310,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'techdocs-cli', @@ -316,6 +322,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'techdocs-container', @@ -327,6 +334,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'techdocs-durp', @@ -336,6 +344,7 @@ describe('GithubDiscoveryProcessor', () => { isFork: false, defaultBranchRef: null, catalogInfoFile: null, + visibility: 'public', }, ], }); @@ -380,6 +389,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'test', @@ -391,6 +401,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'test-archived', @@ -402,6 +413,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'testxyz', @@ -413,6 +425,7 @@ describe('GithubDiscoveryProcessor', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, ], }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts index 21e6a46fbf..6540962411 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts @@ -181,6 +181,7 @@ describe('GithubEntityProvider', () => { id: 'abc123', text: 'some yaml', }, + visibility: 'public', }, ], }), @@ -284,6 +285,7 @@ describe('GithubEntityProvider', () => { id: 'abc123', text: 'some yaml', }, + visibility: 'public', }, ], }), @@ -364,6 +366,7 @@ describe('GithubEntityProvider', () => { id: 'abc123', text: 'some yaml', }, + visibility: 'public', }, ], }), @@ -453,6 +456,7 @@ describe('GithubEntityProvider', () => { name: 'main', }, catalogInfoFile: null, + visibility: 'public', }, { name: 'another-repo', @@ -470,6 +474,7 @@ describe('GithubEntityProvider', () => { id: 'abc123', text: 'some yaml', }, + visibility: 'public', }, ], }), @@ -571,6 +576,7 @@ describe('GithubEntityProvider', () => { id: 'abc123', text: 'some yaml', }, + visibility: 'public', }, { name: 'test-repo-2', @@ -595,6 +601,7 @@ describe('GithubEntityProvider', () => { id: 'abc123', text: 'some yaml', }, + visibility: 'public', }, { name: 'test-repo-3', @@ -616,6 +623,7 @@ describe('GithubEntityProvider', () => { id: 'abc123', text: 'some yaml', }, + visibility: 'public', }, ], }), diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index 839f40e91a..011f4eab6e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -40,7 +40,11 @@ import { GithubEntityProviderConfig, } from './GithubEntityProviderConfig'; import { getOrganizationRepositories } from '../lib/github'; -import { satisfiesTopicFilter, satisfiesForkFilter } from '../lib/util'; +import { + satisfiesTopicFilter, + satisfiesForkFilter, + satisfiesVisibilityFilter, +} from '../lib/util'; import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; import { PushEvent, Commit } from '@octokit/webhooks-types'; @@ -56,6 +60,7 @@ type Repository = { repositoryTopics: string[]; defaultBranchRef?: string; isCatalogInfoFilePresent: boolean; + visibility: string; }; /** @@ -221,6 +226,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { isCatalogInfoFilePresent: r.catalogInfoFile?.__typename === 'Blob' && r.catalogInfoFile.text !== '', + visibility: r.visibility, }; }); @@ -237,6 +243,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { const repositoryFilter = this.config.filters?.repository; const topicFilters = this.config.filters?.topic; const allowForks = this.config.filters?.allowForks ?? true; + const visibilities = this.config.filters?.visibilities ?? []; const matchingRepositories = repositories.filter(r => { const repoTopics: string[] = r.repositoryTopics; @@ -245,6 +252,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { (!repositoryFilter || repositoryFilter.test(r.name)) && satisfiesTopicFilter(repoTopics, topicFilters) && satisfiesForkFilter(allowForks, r.isFork) && + satisfiesVisibilityFilter(visibilities, r.visibility) && r.defaultBranchRef ); }); @@ -310,6 +318,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { // we can consider this file present because // only the catalog file will be recovered from the commits isCatalogInfoFilePresent: true, + visibility: event.repository.visibility, }; const matchingTargets = this.matchesFilters([repository]); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index a597e9dbe0..68091db7e6 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -84,6 +84,12 @@ describe('readProviderConfigs', () => { allowForks: false, }, }, + providerWithVisibilitiesFilter: { + organization: 'test-org6', + filters: { + visibilities: ['public', 'internal'], + }, + }, providerWithHost: { organization: 'test-org1', host: 'ghe.internal.com', @@ -103,7 +109,7 @@ describe('readProviderConfigs', () => { }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(8); + expect(providerConfigs).toHaveLength(9); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -117,6 +123,7 @@ describe('readProviderConfigs', () => { include: undefined, exclude: undefined, }, + visibilities: undefined, }, schedule: undefined, validateLocationsExist: false, @@ -134,6 +141,7 @@ describe('readProviderConfigs', () => { include: undefined, exclude: undefined, }, + visibilities: undefined, }, schedule: undefined, validateLocationsExist: false, @@ -151,6 +159,7 @@ describe('readProviderConfigs', () => { include: undefined, exclude: undefined, }, + visibilities: undefined, }, schedule: undefined, validateLocationsExist: false, @@ -168,6 +177,7 @@ describe('readProviderConfigs', () => { include: undefined, exclude: undefined, }, + visibilities: undefined, }, schedule: undefined, validateLocationsExist: false, @@ -185,6 +195,7 @@ describe('readProviderConfigs', () => { include: ['backstage-include'], exclude: ['backstage-exclude'], }, + visibilities: undefined, }, schedule: undefined, validateLocationsExist: false, @@ -202,11 +213,30 @@ describe('readProviderConfigs', () => { include: undefined, exclude: undefined, }, + visibilities: undefined, }, schedule: undefined, validateLocationsExist: false, }); expect(providerConfigs[6]).toEqual({ + id: 'providerWithVisibilitiesFilter', + organization: 'test-org6', + catalogPath: '/catalog-info.yaml', + host: 'github.com', + filters: { + repository: undefined, + branch: undefined, + allowForks: true, + topic: { + include: undefined, + exclude: undefined, + }, + visibilities: ['public', 'internal'], + }, + schedule: undefined, + validateLocationsExist: false, + }); + expect(providerConfigs[7]).toEqual({ id: 'providerWithHost', organization: 'test-org1', catalogPath: '/catalog-info.yaml', @@ -219,11 +249,12 @@ describe('readProviderConfigs', () => { include: undefined, exclude: undefined, }, + visibilities: undefined, }, validateLocationsExist: false, schedule: undefined, }); - expect(providerConfigs[7]).toEqual({ + expect(providerConfigs[8]).toEqual({ id: 'providerWithSchedule', organization: 'test-org1', catalogPath: '/catalog-info.yaml', @@ -236,6 +267,7 @@ describe('readProviderConfigs', () => { include: undefined, exclude: undefined, }, + visibilities: undefined, }, schedule: { frequency: Duration.fromISO('PT30M'), diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts index a79a557ebf..71e1ba4bb4 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts @@ -33,6 +33,7 @@ export type GithubEntityProviderConfig = { branch?: string; topic?: GithubTopicFilters; allowForks?: boolean; + visibilities?: string[]; }; validateLocationsExist: boolean; schedule?: TaskScheduleDefinition; @@ -85,6 +86,10 @@ function readProviderConfig( const catalogPathContainsWildcard = catalogPath.includes('*'); + const visibilitiesInclude = config?.getOptionalStringArray( + 'filters.visibilities', + ); + if (validateLocationsExist && catalogPathContainsWildcard) { throw Error( `Error while processing GitHub provider config. The catalog path ${catalogPath} contains a wildcard, which is incompatible with validation of locations existing before emitting them. Ensure that validateLocationsExist is set to false.`, @@ -110,6 +115,7 @@ function readProviderConfig( include: topicFilterInclude, exclude: topicFilterExclude, }, + visibilities: visibilitiesInclude, }, schedule, validateLocationsExist,