diff --git a/.changeset/old-parents-cut.md b/.changeset/old-parents-cut.md new file mode 100644 index 0000000000..d2fdaac564 --- /dev/null +++ b/.changeset/old-parents-cut.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Added filter to include archived repositories diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 354569cdb6..697efc9cff 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -168,6 +168,8 @@ If you do so, `default` will be used as provider ID. If configured, all repositories _except_ those with one (or more) topics(s) present in the exclusion filter will be ingested. - **`visibility`** _(optional)_: An array of strings used to filter results based on their visibility. Available options are `private`, `internal`, `public`. If configured (non empty), only repositories with visibility present in the filter will be ingested + - **`allowArchived`** _(optional)_: + Whether to include archived repositories. Defaults to `false`. - **`host`** _(optional)_: The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md). - **`organization`**: diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index c94593b985..f7c8781c30 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -117,6 +117,11 @@ export interface Config { * (Optional) GitHub repository visibility filter. */ visibility?: Array<'private' | 'internal' | 'public'>; + /** + * (Optional) Whether to include archived repositories. + * Default: `false`. + */ + allowArchived?: boolean; }; /** * (Optional) TaskScheduleDefinition for the refresh. @@ -186,6 +191,11 @@ export interface Config { * (Optional) GitHub repository visibility filter. */ visibility?: Array<'private' | 'internal' | 'public'>; + /** + * (Optional) Whether to include archived repositories. + * Default: `false`. + */ + allowArchived?: boolean; }; /** * (Optional) TaskScheduleDefinition for the refresh. diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index e0837de54e..131237c581 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -256,15 +256,16 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { } private matchesFilters(repositories: Repository[]): Repository[] { - const repositoryFilter = this.config.filters?.repository; - const topicFilters = this.config.filters?.topic; - const allowForks = this.config.filters?.allowForks ?? true; - const visibilities = this.config.filters?.visibility ?? []; + const repositoryFilter = this.config.filters.repository; + const topicFilters = this.config.filters.topic; + const allowForks = this.config.filters.allowForks; + const visibilities = this.config.filters.visibility ?? []; + const allowArchived = this.config.filters.allowArchived; return repositories.filter(r => { const repoTopics: string[] = r.repositoryTopics; return ( - !r.isArchived && + (allowArchived || !r.isArchived) && (!repositoryFilter || repositoryFilter.test(r.name)) && satisfiesTopicFilter(repoTopics, topicFilters) && satisfiesForkFilter(allowForks, r.isFork) && @@ -276,7 +277,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { private createLocationUrl(repository: Repository): string { const branch = - this.config.filters?.branch || repository.defaultBranchRef || '-'; + this.config.filters.branch || repository.defaultBranchRef || '-'; const catalogFile = this.config.catalogPath.startsWith('/') ? this.config.catalogPath.substring(1) : this.config.catalogPath; @@ -334,7 +335,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { this.logger.debug(`handle github:push event for ${repoName} - ${repoUrl}`); const branch = - this.config.filters?.branch || event.repository.default_branch; + this.config.filters.branch || event.repository.default_branch; if (!event.ref.includes(branch)) { this.logger.debug(`skipping push event from ref ${event.ref}`); @@ -468,6 +469,8 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { * @param event - The repository archived event. */ private async onRepoArchived(event: RepositoryArchivedEvent) { + if (this.config.filters.allowArchived) return; + const repository = this.createRepoFromEvent(event); await this.removeEntitiesForRepo(repository); this.logger.debug( @@ -549,7 +552,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { * * Creates new entities for the repository if it matches the filters. * - * @param event - The repository unarchived event. + * @param event - The repository transferred event. */ private async onRepoTransferred(event: RepositoryTransferredEvent) { const repository = this.createRepoFromEvent(event); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index bddb4d27cd..28a3f5e7dd 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -92,6 +92,12 @@ describe('readProviderConfigs', () => { visibility: ['public', 'internal'], }, }, + providerWithArchiveFilter: { + organization: 'test-org6', + filters: { + allowArchived: true, + }, + }, providerWithHost: { organization: 'test-org1', host: 'ghe.internal.com', @@ -111,7 +117,7 @@ describe('readProviderConfigs', () => { }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(9); + expect(providerConfigs).toHaveLength(10); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -126,6 +132,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -144,6 +151,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -162,6 +170,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -180,6 +189,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -198,6 +208,7 @@ describe('readProviderConfigs', () => { exclude: ['backstage-exclude'], }, visibility: undefined, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -216,6 +227,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -234,11 +246,31 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: ['public', 'internal'], + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, }); expect(providerConfigs[7]).toEqual({ + id: 'providerWithArchiveFilter', + organization: 'test-org6', + catalogPath: '/catalog-info.yaml', + host: 'github.com', + filters: { + repository: undefined, + branch: undefined, + allowForks: true, + topic: { + include: undefined, + exclude: undefined, + }, + visibility: undefined, + allowArchived: true, + }, + schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, + validateLocationsExist: false, + }); + expect(providerConfigs[8]).toEqual({ id: 'providerWithHost', organization: 'test-org1', catalogPath: '/catalog-info.yaml', @@ -252,11 +284,12 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + allowArchived: false, }, validateLocationsExist: false, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, }); - expect(providerConfigs[8]).toEqual({ + expect(providerConfigs[9]).toEqual({ id: 'providerWithSchedule', organization: 'test-org1', catalogPath: '/catalog-info.yaml', @@ -270,6 +303,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + allowArchived: false, }, schedule: { frequency: { minutes: 30 }, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts index 781239978f..e33e7adfdd 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts @@ -37,12 +37,13 @@ export type GithubEntityProviderConfig = { catalogPath: string; organization: string; host: string; - filters?: { + filters: { repository?: RegExp; branch?: string; topic?: GithubTopicFilters; - allowForks?: boolean; + allowForks: boolean; visibility?: string[]; + allowArchived: boolean; }; validateLocationsExist: boolean; schedule?: SchedulerServiceTaskScheduleDefinition; @@ -90,6 +91,8 @@ function readProviderConfig( const topicFilterExclude = config?.getOptionalStringArray( 'filters.topic.exclude', ); + const allowArchived = + config.getOptionalBoolean('filters.allowArchived') ?? false; const validateLocationsExist = config?.getOptionalBoolean('validateLocationsExist') ?? false; @@ -132,6 +135,7 @@ function readProviderConfig( exclude: topicFilterExclude, }, visibility: visibilityFilterInclude, + allowArchived, }, schedule, validateLocationsExist,