From 2fe1f5973ff7592b45d05b632cbe9af8c1387c0d Mon Sep 17 00:00:00 2001 From: alessandro Date: Thu, 27 Jul 2023 15:10:03 +0200 Subject: [PATCH 1/3] feat(catalog-backend-module-gitlab): Filter Gitlab archived projects through APIs Signed-off-by: alessandro --- .changeset/serious-papayas-shout.md | 5 +++++ .../src/GitLabDiscoveryProcessor.ts | 5 +---- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 1 + .../src/providers/GitlabDiscoveryEntityProvider.ts | 5 +---- 4 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 .changeset/serious-papayas-shout.md diff --git a/.changeset/serious-papayas-shout.md b/.changeset/serious-papayas-shout.md new file mode 100644 index 0000000000..f426dd02c7 --- /dev/null +++ b/.changeset/serious-papayas-shout.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Filter Gitlab archived projects through APIs diff --git a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts index c9bd4d9187..02b1b1cb51 100644 --- a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts @@ -109,6 +109,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { const lastActivity = (await this.cache.get(this.getCacheKey())) as string; const opts = { + archived: false, group, page: 1, // We check for the existence of lastActivity and only set it if it's present to ensure @@ -125,10 +126,6 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { for await (const project of projects) { res.scanned++; - if (project.archived) { - continue; - } - if (branch === '*' && project.default_branch === undefined) { continue; } diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index f0e169d8df..41b99e1c5b 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -30,6 +30,7 @@ export type CommonListOptions = { }; interface ListProjectOptions extends CommonListOptions { + archived?: boolean; group?: string; } diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index 08d2fe48a7..ce7a304746 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -155,6 +155,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { const projects = paginated( options => client.listProjects(options), { + archived: false, group: this.config.group, page: 1, per_page: 50, @@ -173,10 +174,6 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { res.scanned++; - if (project.archived) { - continue; - } - if ( this.config.skipForkedRepos && project.hasOwnProperty('forked_from_project') From 6550e48cb5a9ed6687bfe4beccf31c50455761ce Mon Sep 17 00:00:00 2001 From: alessandro Date: Mon, 7 Aug 2023 09:55:15 +0200 Subject: [PATCH 2/3] feat(catalog-backend-module-gitlab): Fix tests for archived projects Signed-off-by: alessandro --- .../src/GitLabDiscoveryProcessor.test.ts | 43 +++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts index 6333184ea8..8a8ceccaf0 100644 --- a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts @@ -56,6 +56,7 @@ function setupFakeServer( listProjectsCallback: (request: { page: number; include_subgroups: boolean; + archived: boolean; }) => { data: GitLabProject[]; nextPage?: number; @@ -74,9 +75,11 @@ function setupFakeServer( } const page = req.url.searchParams.get('page'); const include_subgroups = req.url.searchParams.get('include_subgroups'); + const archived = req.url.searchParams.get('archived'); const response = listProjectsCallback({ page: parseInt(page!, 10), include_subgroups: include_subgroups === 'true', + archived: archived === 'true', }); // Filter the fake results based on the `last_activity_after` parameter @@ -222,6 +225,37 @@ describe('GitlabDiscoveryProcessor', () => { nextPage: 2, }; case 2: + if (request.archived) { + return { + data: [ + { + id: 3, + archived: false, + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/3', + path_with_namespace: '3', + }, + { + id: 4, + archived: true, // ARCHIVED + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/4', + path_with_namespace: '4', + }, + { + id: 5, + archived: false, + default_branch: undefined, // MISSING DEFAULT BRANCH + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/g/5', + path_with_namespace: 'g/5', + }, + ], + }; + } + return { data: [ { @@ -232,14 +266,6 @@ describe('GitlabDiscoveryProcessor', () => { web_url: 'https://gitlab.fake/3', path_with_namespace: '3', }, - { - id: 4, - archived: true, // ARCHIVED - default_branch: 'master', - last_activity_at: '2021-08-05T11:03:05.774Z', - web_url: 'https://gitlab.fake/4', - path_with_namespace: '4', - }, { id: 5, archived: false, @@ -250,6 +276,7 @@ describe('GitlabDiscoveryProcessor', () => { }, ], }; + default: throw new Error('Invalid request'); } From 919b52f7dfdb3a28d159d73f927c1cd325f5e017 Mon Sep 17 00:00:00 2001 From: alessandro Date: Mon, 7 Aug 2023 10:02:22 +0200 Subject: [PATCH 3/3] feat(catalog-backend-module-gitlab): Fix tests for archived projects Signed-off-by: alessandro --- .../src/GitLabDiscoveryProcessor.test.ts | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts index 8a8ceccaf0..a56c229105 100644 --- a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts @@ -86,11 +86,13 @@ function setupFakeServer( const last_activity_after = req.url.searchParams.get( 'last_activity_after', ); - const filteredData = response.data.filter( - v => - !last_activity_after || - Date.parse(v.last_activity_at) >= Date.parse(last_activity_after), - ); + const filteredData = response.data + .filter( + v => + !last_activity_after || + Date.parse(v.last_activity_at) >= Date.parse(last_activity_after), + ) + .filter(v => archived || !v.archived); return res( ctx.set('x-next-page', response.nextPage?.toString() ?? ''), @@ -225,37 +227,6 @@ describe('GitlabDiscoveryProcessor', () => { nextPage: 2, }; case 2: - if (request.archived) { - return { - data: [ - { - id: 3, - archived: false, - default_branch: 'master', - last_activity_at: '2021-08-05T11:03:05.774Z', - web_url: 'https://gitlab.fake/3', - path_with_namespace: '3', - }, - { - id: 4, - archived: true, // ARCHIVED - default_branch: 'master', - last_activity_at: '2021-08-05T11:03:05.774Z', - web_url: 'https://gitlab.fake/4', - path_with_namespace: '4', - }, - { - id: 5, - archived: false, - default_branch: undefined, // MISSING DEFAULT BRANCH - last_activity_at: '2021-08-05T11:03:05.774Z', - web_url: 'https://gitlab.fake/g/5', - path_with_namespace: 'g/5', - }, - ], - }; - } - return { data: [ { @@ -266,6 +237,14 @@ describe('GitlabDiscoveryProcessor', () => { web_url: 'https://gitlab.fake/3', path_with_namespace: '3', }, + { + id: 4, + archived: true, // ARCHIVED + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/4', + path_with_namespace: '4', + }, { id: 5, archived: false,