diff --git a/.changeset/old-parents-cut.md b/.changeset/old-parents-cut.md new file mode 100644 index 0000000000..d2fdaac564 --- /dev/null +++ b/.changeset/old-parents-cut.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Added filter to include archived repositories diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 354569cdb6..1c45d12f83 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -168,6 +168,8 @@ If you do so, `default` will be used as provider ID. If configured, all repositories _except_ those with one (or more) topics(s) present in the exclusion filter will be ingested. - **`visibility`** _(optional)_: An array of strings used to filter results based on their visibility. Available options are `private`, `internal`, `public`. If configured (non empty), only repositories with visibility present in the filter will be ingested + - **`includeArchived`** _(optional)_: + Whether to include archived repositories. Defaults to `false`. - **`host`** _(optional)_: The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md). - **`organization`**: diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index c94593b985..976256ac9e 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -117,6 +117,11 @@ export interface Config { * (Optional) GitHub repository visibility filter. */ visibility?: Array<'private' | 'internal' | 'public'>; + /** + * (Optional) Whether to include archived repositories. + * Default: `false`. + */ + includeArchived?: boolean; }; /** * (Optional) TaskScheduleDefinition for the refresh. @@ -186,6 +191,11 @@ export interface Config { * (Optional) GitHub repository visibility filter. */ visibility?: Array<'private' | 'internal' | 'public'>; + /** + * (Optional) Whether to include archived repositories. + * Default: `false`. + */ + includeArchived?: boolean; }; /** * (Optional) TaskScheduleDefinition for the refresh. diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index e0837de54e..7790be56a0 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -256,15 +256,16 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { } private matchesFilters(repositories: Repository[]): Repository[] { - const repositoryFilter = this.config.filters?.repository; - const topicFilters = this.config.filters?.topic; - const allowForks = this.config.filters?.allowForks ?? true; - const visibilities = this.config.filters?.visibility ?? []; + const repositoryFilter = this.config.filters.repository; + const topicFilters = this.config.filters.topic; + const allowForks = this.config.filters.allowForks; + const visibilities = this.config.filters.visibility ?? []; + const includeArchived = this.config.filters.includeArchived; return repositories.filter(r => { const repoTopics: string[] = r.repositoryTopics; return ( - !r.isArchived && + (includeArchived || !r.isArchived) && (!repositoryFilter || repositoryFilter.test(r.name)) && satisfiesTopicFilter(repoTopics, topicFilters) && satisfiesForkFilter(allowForks, r.isFork) && @@ -276,7 +277,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { private createLocationUrl(repository: Repository): string { const branch = - this.config.filters?.branch || repository.defaultBranchRef || '-'; + this.config.filters.branch || repository.defaultBranchRef || '-'; const catalogFile = this.config.catalogPath.startsWith('/') ? this.config.catalogPath.substring(1) : this.config.catalogPath; @@ -334,7 +335,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { this.logger.debug(`handle github:push event for ${repoName} - ${repoUrl}`); const branch = - this.config.filters?.branch || event.repository.default_branch; + this.config.filters.branch || event.repository.default_branch; if (!event.ref.includes(branch)) { this.logger.debug(`skipping push event from ref ${event.ref}`); @@ -468,6 +469,8 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { * @param event - The repository archived event. */ private async onRepoArchived(event: RepositoryArchivedEvent) { + if (this.config.filters.includeArchived) return; + const repository = this.createRepoFromEvent(event); await this.removeEntitiesForRepo(repository); this.logger.debug( @@ -549,7 +552,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { * * Creates new entities for the repository if it matches the filters. * - * @param event - The repository unarchived event. + * @param event - The repository transferred event. */ private async onRepoTransferred(event: RepositoryTransferredEvent) { const repository = this.createRepoFromEvent(event); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index bddb4d27cd..0edde80252 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -126,6 +126,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -144,6 +145,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -162,6 +164,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -180,6 +183,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -198,6 +202,7 @@ describe('readProviderConfigs', () => { exclude: ['backstage-exclude'], }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -216,6 +221,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -234,6 +240,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: ['public', 'internal'], + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -252,6 +259,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, validateLocationsExist: false, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, @@ -270,6 +278,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: { frequency: { minutes: 30 }, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts index 781239978f..9004444f28 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts @@ -37,12 +37,13 @@ export type GithubEntityProviderConfig = { catalogPath: string; organization: string; host: string; - filters?: { + filters: { repository?: RegExp; branch?: string; topic?: GithubTopicFilters; - allowForks?: boolean; + allowForks: boolean; visibility?: string[]; + includeArchived: boolean; }; validateLocationsExist: boolean; schedule?: SchedulerServiceTaskScheduleDefinition; @@ -90,6 +91,8 @@ function readProviderConfig( const topicFilterExclude = config?.getOptionalStringArray( 'filters.topic.exclude', ); + const includeArchived = + config.getOptionalBoolean('filters.includeArchived') ?? false; const validateLocationsExist = config?.getOptionalBoolean('validateLocationsExist') ?? false; @@ -132,6 +135,7 @@ function readProviderConfig( exclude: topicFilterExclude, }, visibility: visibilityFilterInclude, + includeArchived, }, schedule, validateLocationsExist,