From ee9f59f3e0011f1ff54bdd9d65312c3992d486e0 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Fri, 9 May 2025 11:47:59 +0200 Subject: [PATCH 1/3] feat(catalog): add filter to include archived repositories Signed-off-by: Benjamin Janssens --- .changeset/old-parents-cut.md | 5 +++++ docs/integrations/github/discovery.md | 2 ++ .../catalog-backend-module-github/config.d.ts | 10 ++++++++++ .../src/providers/GithubEntityProvider.ts | 19 +++++++++++-------- .../GithubEntityProviderConfig.test.ts | 9 +++++++++ .../providers/GithubEntityProviderConfig.ts | 8 ++++++-- 6 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 .changeset/old-parents-cut.md diff --git a/.changeset/old-parents-cut.md b/.changeset/old-parents-cut.md new file mode 100644 index 0000000000..d2fdaac564 --- /dev/null +++ b/.changeset/old-parents-cut.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Added filter to include archived repositories diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 354569cdb6..1c45d12f83 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -168,6 +168,8 @@ If you do so, `default` will be used as provider ID. If configured, all repositories _except_ those with one (or more) topics(s) present in the exclusion filter will be ingested. - **`visibility`** _(optional)_: An array of strings used to filter results based on their visibility. Available options are `private`, `internal`, `public`. If configured (non empty), only repositories with visibility present in the filter will be ingested + - **`includeArchived`** _(optional)_: + Whether to include archived repositories. Defaults to `false`. - **`host`** _(optional)_: The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md). - **`organization`**: diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index c94593b985..976256ac9e 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -117,6 +117,11 @@ export interface Config { * (Optional) GitHub repository visibility filter. */ visibility?: Array<'private' | 'internal' | 'public'>; + /** + * (Optional) Whether to include archived repositories. + * Default: `false`. + */ + includeArchived?: boolean; }; /** * (Optional) TaskScheduleDefinition for the refresh. @@ -186,6 +191,11 @@ export interface Config { * (Optional) GitHub repository visibility filter. */ visibility?: Array<'private' | 'internal' | 'public'>; + /** + * (Optional) Whether to include archived repositories. + * Default: `false`. + */ + includeArchived?: boolean; }; /** * (Optional) TaskScheduleDefinition for the refresh. diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index e0837de54e..7790be56a0 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -256,15 +256,16 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { } private matchesFilters(repositories: Repository[]): Repository[] { - const repositoryFilter = this.config.filters?.repository; - const topicFilters = this.config.filters?.topic; - const allowForks = this.config.filters?.allowForks ?? true; - const visibilities = this.config.filters?.visibility ?? []; + const repositoryFilter = this.config.filters.repository; + const topicFilters = this.config.filters.topic; + const allowForks = this.config.filters.allowForks; + const visibilities = this.config.filters.visibility ?? []; + const includeArchived = this.config.filters.includeArchived; return repositories.filter(r => { const repoTopics: string[] = r.repositoryTopics; return ( - !r.isArchived && + (includeArchived || !r.isArchived) && (!repositoryFilter || repositoryFilter.test(r.name)) && satisfiesTopicFilter(repoTopics, topicFilters) && satisfiesForkFilter(allowForks, r.isFork) && @@ -276,7 +277,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { private createLocationUrl(repository: Repository): string { const branch = - this.config.filters?.branch || repository.defaultBranchRef || '-'; + this.config.filters.branch || repository.defaultBranchRef || '-'; const catalogFile = this.config.catalogPath.startsWith('/') ? this.config.catalogPath.substring(1) : this.config.catalogPath; @@ -334,7 +335,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { this.logger.debug(`handle github:push event for ${repoName} - ${repoUrl}`); const branch = - this.config.filters?.branch || event.repository.default_branch; + this.config.filters.branch || event.repository.default_branch; if (!event.ref.includes(branch)) { this.logger.debug(`skipping push event from ref ${event.ref}`); @@ -468,6 +469,8 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { * @param event - The repository archived event. */ private async onRepoArchived(event: RepositoryArchivedEvent) { + if (this.config.filters.includeArchived) return; + const repository = this.createRepoFromEvent(event); await this.removeEntitiesForRepo(repository); this.logger.debug( @@ -549,7 +552,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { * * Creates new entities for the repository if it matches the filters. * - * @param event - The repository unarchived event. + * @param event - The repository transferred event. */ private async onRepoTransferred(event: RepositoryTransferredEvent) { const repository = this.createRepoFromEvent(event); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index bddb4d27cd..0edde80252 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -126,6 +126,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -144,6 +145,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -162,6 +164,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -180,6 +183,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -198,6 +202,7 @@ describe('readProviderConfigs', () => { exclude: ['backstage-exclude'], }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -216,6 +221,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -234,6 +240,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: ['public', 'internal'], + includeArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -252,6 +259,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, validateLocationsExist: false, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, @@ -270,6 +278,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, + includeArchived: false, }, schedule: { frequency: { minutes: 30 }, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts index 781239978f..9004444f28 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts @@ -37,12 +37,13 @@ export type GithubEntityProviderConfig = { catalogPath: string; organization: string; host: string; - filters?: { + filters: { repository?: RegExp; branch?: string; topic?: GithubTopicFilters; - allowForks?: boolean; + allowForks: boolean; visibility?: string[]; + includeArchived: boolean; }; validateLocationsExist: boolean; schedule?: SchedulerServiceTaskScheduleDefinition; @@ -90,6 +91,8 @@ function readProviderConfig( const topicFilterExclude = config?.getOptionalStringArray( 'filters.topic.exclude', ); + const includeArchived = + config.getOptionalBoolean('filters.includeArchived') ?? false; const validateLocationsExist = config?.getOptionalBoolean('validateLocationsExist') ?? false; @@ -132,6 +135,7 @@ function readProviderConfig( exclude: topicFilterExclude, }, visibility: visibilityFilterInclude, + includeArchived, }, schedule, validateLocationsExist, From 9e6c479fddee34fe94a5ecccf9ae078a826704c5 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Fri, 9 May 2025 11:57:57 +0200 Subject: [PATCH 2/3] test(catalog:) extend existing tests Signed-off-by: Benjamin Janssens --- .../GithubEntityProviderConfig.test.ts | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index 0edde80252..9ed1bb2e27 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -92,6 +92,12 @@ describe('readProviderConfigs', () => { visibility: ['public', 'internal'], }, }, + providerWithArchiveFilter: { + organization: 'test-org6', + filters: { + includeArchived: true, + }, + }, providerWithHost: { organization: 'test-org1', host: 'ghe.internal.com', @@ -111,7 +117,7 @@ describe('readProviderConfigs', () => { }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(9); + expect(providerConfigs).toHaveLength(10); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -246,6 +252,25 @@ describe('readProviderConfigs', () => { validateLocationsExist: false, }); expect(providerConfigs[7]).toEqual({ + id: 'providerWithArchiveFilter', + organization: 'test-org6', + catalogPath: '/catalog-info.yaml', + host: 'github.com', + filters: { + repository: undefined, + branch: undefined, + allowForks: true, + topic: { + include: undefined, + exclude: undefined, + }, + visibility: undefined, + includeArchived: true, + }, + schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, + validateLocationsExist: false, + }); + expect(providerConfigs[8]).toEqual({ id: 'providerWithHost', organization: 'test-org1', catalogPath: '/catalog-info.yaml', @@ -264,7 +289,7 @@ describe('readProviderConfigs', () => { validateLocationsExist: false, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, }); - expect(providerConfigs[8]).toEqual({ + expect(providerConfigs[9]).toEqual({ id: 'providerWithSchedule', organization: 'test-org1', catalogPath: '/catalog-info.yaml', From bc7751a74ec33530312b62cd9d671c61960e210b Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Mon, 12 May 2025 18:00:59 +0200 Subject: [PATCH 3/3] chore(catalog): rename includeArchived to allowArchived Signed-off-by: Benjamin Janssens --- docs/integrations/github/discovery.md | 2 +- .../catalog-backend-module-github/config.d.ts | 4 ++-- .../src/providers/GithubEntityProvider.ts | 6 ++--- .../GithubEntityProviderConfig.test.ts | 22 +++++++++---------- .../providers/GithubEntityProviderConfig.ts | 8 +++---- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 1c45d12f83..697efc9cff 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -168,7 +168,7 @@ If you do so, `default` will be used as provider ID. If configured, all repositories _except_ those with one (or more) topics(s) present in the exclusion filter will be ingested. - **`visibility`** _(optional)_: An array of strings used to filter results based on their visibility. Available options are `private`, `internal`, `public`. If configured (non empty), only repositories with visibility present in the filter will be ingested - - **`includeArchived`** _(optional)_: + - **`allowArchived`** _(optional)_: Whether to include archived repositories. Defaults to `false`. - **`host`** _(optional)_: The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md). diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index 976256ac9e..f7c8781c30 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -121,7 +121,7 @@ export interface Config { * (Optional) Whether to include archived repositories. * Default: `false`. */ - includeArchived?: boolean; + allowArchived?: boolean; }; /** * (Optional) TaskScheduleDefinition for the refresh. @@ -195,7 +195,7 @@ export interface Config { * (Optional) Whether to include archived repositories. * Default: `false`. */ - includeArchived?: boolean; + allowArchived?: boolean; }; /** * (Optional) TaskScheduleDefinition for the refresh. diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index 7790be56a0..131237c581 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -260,12 +260,12 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { const topicFilters = this.config.filters.topic; const allowForks = this.config.filters.allowForks; const visibilities = this.config.filters.visibility ?? []; - const includeArchived = this.config.filters.includeArchived; + const allowArchived = this.config.filters.allowArchived; return repositories.filter(r => { const repoTopics: string[] = r.repositoryTopics; return ( - (includeArchived || !r.isArchived) && + (allowArchived || !r.isArchived) && (!repositoryFilter || repositoryFilter.test(r.name)) && satisfiesTopicFilter(repoTopics, topicFilters) && satisfiesForkFilter(allowForks, r.isFork) && @@ -469,7 +469,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { * @param event - The repository archived event. */ private async onRepoArchived(event: RepositoryArchivedEvent) { - if (this.config.filters.includeArchived) return; + if (this.config.filters.allowArchived) return; const repository = this.createRepoFromEvent(event); await this.removeEntitiesForRepo(repository); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index 9ed1bb2e27..28a3f5e7dd 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -95,7 +95,7 @@ describe('readProviderConfigs', () => { providerWithArchiveFilter: { organization: 'test-org6', filters: { - includeArchived: true, + allowArchived: true, }, }, providerWithHost: { @@ -132,7 +132,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, - includeArchived: false, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -151,7 +151,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, - includeArchived: false, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -170,7 +170,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, - includeArchived: false, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -189,7 +189,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, - includeArchived: false, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -208,7 +208,7 @@ describe('readProviderConfigs', () => { exclude: ['backstage-exclude'], }, visibility: undefined, - includeArchived: false, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -227,7 +227,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, - includeArchived: false, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -246,7 +246,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: ['public', 'internal'], - includeArchived: false, + allowArchived: false, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -265,7 +265,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, - includeArchived: true, + allowArchived: true, }, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, validateLocationsExist: false, @@ -284,7 +284,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, - includeArchived: false, + allowArchived: false, }, validateLocationsExist: false, schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, @@ -303,7 +303,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, visibility: undefined, - includeArchived: false, + allowArchived: false, }, schedule: { frequency: { minutes: 30 }, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts index 9004444f28..e33e7adfdd 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts @@ -43,7 +43,7 @@ export type GithubEntityProviderConfig = { topic?: GithubTopicFilters; allowForks: boolean; visibility?: string[]; - includeArchived: boolean; + allowArchived: boolean; }; validateLocationsExist: boolean; schedule?: SchedulerServiceTaskScheduleDefinition; @@ -91,8 +91,8 @@ function readProviderConfig( const topicFilterExclude = config?.getOptionalStringArray( 'filters.topic.exclude', ); - const includeArchived = - config.getOptionalBoolean('filters.includeArchived') ?? false; + const allowArchived = + config.getOptionalBoolean('filters.allowArchived') ?? false; const validateLocationsExist = config?.getOptionalBoolean('validateLocationsExist') ?? false; @@ -135,7 +135,7 @@ function readProviderConfig( exclude: topicFilterExclude, }, visibility: visibilityFilterInclude, - includeArchived, + allowArchived, }, schedule, validateLocationsExist,