From 1b5fdd97a9347d547bb68c824c49131acee15052 Mon Sep 17 00:00:00 2001 From: Mkolaj Kaliszewski Date: Fri, 18 Oct 2024 11:23:59 +0200 Subject: [PATCH] Add support for includeArchivedRepos in catalog-backend-module-gitlab config, update docs Signed-off-by: Mkolaj Kaliszewski --- .changeset/silver-kings-sort.md | 5 + docs/integrations/gitlab/discovery.md | 3 + .../catalog-backend-module-gitlab/config.d.ts | 4 + .../src/GitLabDiscoveryProcessor.ts | 6 +- .../src/__testUtils__/handlers.ts | 35 +++- .../src/__testUtils__/mocks.ts | 162 +++++++++++++----- .../src/lib/client.test.ts | 26 +++ .../src/lib/types.ts | 4 + .../GitlabDiscoveryEntityProvider.test.ts | 29 ++++ .../GitlabDiscoveryEntityProvider.ts | 2 +- .../src/providers/config.test.ts | 49 ++++++ .../src/providers/config.ts | 4 + 12 files changed, 274 insertions(+), 55 deletions(-) create mode 100644 .changeset/silver-kings-sort.md diff --git a/.changeset/silver-kings-sort.md b/.changeset/silver-kings-sort.md new file mode 100644 index 0000000000..3cfbeb68ce --- /dev/null +++ b/.changeset/silver-kings-sort.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': minor +--- + +Extended the configuration with the includeArchivedRepos property, which allows including repositories when the project is archived. diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index eab590ff11..de35fbcc96 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -150,6 +150,7 @@ catalog: branch: main # Optional. Used to discover on a specific branch fallbackBranch: master # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default skipForkedRepos: false # Optional. If the project is a fork, skip repository + includeArchivedRepos: false # Optional. If project is archived, include repository group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything @@ -212,3 +213,5 @@ catalog: If you don't want create location object if file with component definition do not exists in project, you can set the `skipReposWithoutExactFileMatch` option. That can reduce count of request to gitlab with 404 status code. If you don't want to create location object if the project is a fork, you can set the `skipForkedRepos` option. + +If you want to create location object if the project is archived, you can set the `includeArchivedRepos` option. diff --git a/plugins/catalog-backend-module-gitlab/config.d.ts b/plugins/catalog-backend-module-gitlab/config.d.ts index e249f13d4c..ce1cda50e7 100644 --- a/plugins/catalog-backend-module-gitlab/config.d.ts +++ b/plugins/catalog-backend-module-gitlab/config.d.ts @@ -63,6 +63,10 @@ export interface Config { * (Optional) Skip forked repository */ skipForkedRepos?: boolean; + /** + * (Optional) Include archived repository + */ + includeArchivedRepos?: boolean; /** * (Optional) A list of strings containing the paths of the repositories to skip * Should be in the format group/subgroup/repo, with no leading or trailing slashes. diff --git a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts index 01e8018845..ab31a44bbc 100644 --- a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts @@ -39,6 +39,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { private readonly cache: CacheService; private readonly skipReposWithoutExactFileMatch: boolean; private readonly skipForkedRepos: boolean; + private readonly includeArchivedRepos: boolean; static fromConfig( config: Config, @@ -46,6 +47,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { logger: LoggerService; skipReposWithoutExactFileMatch?: boolean; skipForkedRepos?: boolean; + includeArchivedRepos?: boolean; }, ): GitLabDiscoveryProcessor { const integrations = ScmIntegrations.fromConfig(config); @@ -65,6 +67,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { logger: LoggerService; skipReposWithoutExactFileMatch?: boolean; skipForkedRepos?: boolean; + includeArchivedRepos?: boolean; }) { this.integrations = options.integrations; this.cache = options.pluginCache; @@ -72,6 +75,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { this.skipReposWithoutExactFileMatch = options.skipReposWithoutExactFileMatch || false; this.skipForkedRepos = options.skipForkedRepos || false; + this.includeArchivedRepos = options.includeArchivedRepos || false; } getProcessorName(): string { @@ -105,12 +109,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { const lastActivity = (await this.cache.get(this.getCacheKey())) as string; const opts = { - archived: false, group, page: 1, // We check for the existence of lastActivity and only set it if it's present to ensure // that the options doesn't include the key so that the API doesn't receive an empty query parameter. ...(lastActivity && { last_activity_after: lastActivity }), + ...(!this.includeArchivedRepos && { archived: false }), }; const projects = paginated(options => client.listProjects(options), opts); diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts index e49fb54465..953ade6b7c 100644 --- a/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts @@ -37,9 +37,18 @@ const httpHandlers = [ * Project REST endpoint mocks */ - // fetch all projects in an instance - rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => { - return res(ctx.set('x-next-page', ''), ctx.json(all_projects_response)); + // fetch all projects in an instance handling archived ones + rest.get(`${apiBaseUrl}/projects`, (req, res, ctx) => { + const archived = req.url.searchParams.get('archived'); + + return res( + ctx.set('x-next-page', ''), + ctx.json( + all_projects_response.filter(p => + archived === 'false' ? !p.archived : true, + ), + ), + ); }), rest.get(`${apiBaseUrl}/projects/42`, (_, res, ctx) => { @@ -136,9 +145,13 @@ const httpGroupFindByIdDynamic = all_groups_response.map(group => { const httpGroupListDescendantProjectsById = all_groups_response.map(group => { return rest.get( `${apiBaseUrl}/groups/${group.id}/projects`, - (_, res, ctx) => { - const projectsInGroup = all_projects_response.filter(p => - p.path_with_namespace?.includes(group.name), + (req, res, ctx) => { + const archived = req.url.searchParams.get('archived'); + + const projectsInGroup = all_projects_response.filter( + p => + p.path_with_namespace?.includes(group.name) && + (archived === 'false' ? !p.archived : true), ); return res(ctx.json(projectsInGroup)); @@ -149,9 +162,13 @@ const httpGroupListDescendantProjectsById = all_groups_response.map(group => { const httpGroupListDescendantProjectsByName = all_groups_response.map(group => { return rest.get( `${apiBaseUrl}/groups/${group.name}/projects`, - (_, res, ctx) => { - const projectsInGroup = all_projects_response.filter(p => - p.path_with_namespace?.includes(group.name), + (req, res, ctx) => { + const archived = req.url.searchParams.get('archived'); + + const projectsInGroup = all_projects_response.filter( + p => + p.path_with_namespace?.includes(group.name) && + (archived === 'false' ? !p.archived : true), ); return res(ctx.json(projectsInGroup)); diff --git a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts index 3087d10f11..2547015df8 100644 --- a/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts +++ b/plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts @@ -299,6 +299,33 @@ export const config_single_integration_skip_forks: MockObject = { }, }; +export const config_single_integration_include_archived: MockObject = { + integrations: { + gitlab: [ + { + host: 'example.com', + apiBaseUrl: 'https://example.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'example.com', + group: 'group1', + includeArchivedRepos: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, +}; + export const config_single_integration_exclude_repos: MockObject = { integrations: { gitlab: [ @@ -856,6 +883,18 @@ export const all_projects_response: GitLabProject[] = [ web_url: 'https://example.com/group1/test-repo7', path_with_namespace: 'group1/test-repo7', }, + // archived project + { + id: 8, + description: 'Project Eight Description', + name: 'test-repo8-archived', + default_branch: 'main', + path: 'test-repo8-archived', + archived: true, + last_activity_at: new Date().toString(), + web_url: 'https://example.com/group1/test-repo8-archived', + path_with_namespace: 'group1/test-repo8-archived', + }, ]; export const all_users_response: GitLabUser[] = [ @@ -1553,7 +1592,7 @@ export const push_modif_event: EventParams = { // includes only projects that have a default branch (for when the branch and fallback branch were not set in the config) export const expected_location_entities_default_branch: MockObject[] = all_projects_response - .filter(project => project.default_branch) + .filter(project => project.default_branch && !project.archived) .map(project => { const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${project.default_branch}/catalog-info.yaml`; @@ -1583,63 +1622,98 @@ export const expected_location_entities_default_branch: MockObject[] = // includes every GitLab project that has a default branch and the fallback declared in the config export const expected_location_entities_fallback_branch: MockObject[] = - all_projects_response.map(project => { - const branch = project.default_branch || 'main'; - const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${branch}/catalog-info.yaml`; + all_projects_response + .filter(project => !project.archived) + .map(project => { + const branch = project.default_branch || 'main'; + const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${branch}/catalog-info.yaml`; - return { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': `url:${targetUrl}`, - 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ + target: targetUrl, + type: 'url', + }), }, - name: locationSpecToMetadataName({ + spec: { + presence: 'optional', target: targetUrl, type: 'url', - }), + }, }, - spec: { - presence: 'optional', - target: targetUrl, - type: 'url', - }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }; - }); + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }); // includes ONLY the projects with the branch declared in the config export const expected_location_entities_specific_branch: MockObject[] = - all_projects_response.map(project => { - const branch = 'develop'; - const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${branch}/catalog-info.yaml`; + all_projects_response + .filter(project => !project.archived) + .map(project => { + const branch = 'develop'; + const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${branch}/catalog-info.yaml`; - return { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': `url:${targetUrl}`, - 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ + target: targetUrl, + type: 'url', + }), }, - name: locationSpecToMetadataName({ + spec: { + presence: 'optional', target: targetUrl, type: 'url', - }), + }, }, - spec: { - presence: 'optional', - target: targetUrl, - type: 'url', + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }); + +// includes archived and not archived projects +export const expected_location_entities_including_archived: MockObject[] = + all_projects_response + .filter(project => project.default_branch) + .map(project => { + const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${project.default_branch}/catalog-info.yaml`; + + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${targetUrl}`, + 'backstage.io/managed-by-origin-location': `url:${targetUrl}`, + }, + name: locationSpecToMetadataName({ + target: targetUrl, + type: 'url', + }), + }, + spec: { + presence: 'optional', + target: targetUrl, + type: 'url', + }, }, - }, - locationKey: 'GitlabDiscoveryEntityProvider:test-id', - }; - }); + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }; + }); export const expected_added_location_entities: MockObject[] = added_commits.map( commit => { diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 1eebe7d6e0..daea71060f 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -143,6 +143,31 @@ describe('GitLabClient', () => { expect(allItems).toHaveLength(expectedProjects.length); }); + it('should get not archived projects', async () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader(mock.config_self_managed), + ), + logger: mockServices.logger.mock(), + }); + + const archivedProjectsGen = paginated( + options => client.listProjects(options), + { archived: false }, + ); + + const expectedProjects = mock.all_projects_response.filter( + project => !project.archived, + ); + + const allItems = []; + for await (const item of archivedProjectsGen) { + allItems.push(item); + } + + expect(allItems).toHaveLength(expectedProjects.length); + }); + it('should get all projects for an instance', async () => { const client = new GitLabClient({ config: readGitLabIntegrationConfig( @@ -159,6 +184,7 @@ describe('GitLabClient', () => { for await (const project of instanceProjects) { allProjects.push(project); } + expect(allProjects).toHaveLength(mock.all_projects_response.length); }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 423f3ff8b5..e25f9bacf4 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -232,6 +232,10 @@ export type GitlabProviderConfig = { * If the project is a fork, skip repository */ skipForkedRepos?: boolean; + /** + * If the project is archived, include repository + */ + includeArchivedRepos?: boolean; /** * List of repositories to exclude from discovery, should be the full path to the repository, e.g. `group/project` * Paths should not start or end with a slash. diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index e6309a8516..dd7a4f0d6d 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -232,6 +232,35 @@ describe('GitlabDiscoveryEntityProvider - refresh', () => { }); }); + it('should include archived projects', async () => { + const config = new ConfigReader( + mock.config_single_integration_include_archived, + ); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + await provider.connect(entityProviderConnection); + + await provider.refresh(logger); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: mock.expected_location_entities_including_archived.filter( + entity => + !entity.entity.metadata.annotations[ + 'backstage.io/managed-by-location' + ].includes('awesome'), + ), + }); + }); + it('should filter repositories that are excluded', async () => { const config = new ConfigReader( mock.config_single_integration_exclude_repos, diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index 0ac45b93e1..66a6247295 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -210,10 +210,10 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { const projects = paginated( options => this.gitLabClient.listProjects(options), { - archived: false, group: this.config.group, page: 1, per_page: 50, + ...(!this.config.includeArchivedRepos && { archived: false }), }, ); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index 4b7c73ec80..e1798df184 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -61,6 +61,7 @@ describe('config', () => { relations: [], schedule: undefined, skipForkedRepos: false, + includeArchivedRepos: false, excludeRepos: [], restrictUsersToGroup: false, includeUsersWithoutSeat: false, @@ -104,6 +105,7 @@ describe('config', () => { relations: [], schedule: undefined, skipForkedRepos: false, + includeArchivedRepos: false, excludeRepos: [], restrictUsersToGroup: false, includeUsersWithoutSeat: true, @@ -149,6 +151,51 @@ describe('config', () => { restrictUsersToGroup: false, excludeRepos: [], skipForkedRepos: true, + includeArchivedRepos: false, + includeUsersWithoutSeat: false, + }), + ); + }); + + it('valid config with includeArchivedRepos', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + group: 'group', + host: 'host', + branch: 'not-master', + fallbackBranch: 'main', + entityFilename: 'custom-file.yaml', + includeArchivedRepos: true, + }, + }, + }, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(1); + result.forEach(r => + expect(r).toStrictEqual({ + id: 'test', + group: 'group', + branch: 'not-master', + fallbackBranch: 'main', + host: 'host', + catalogFile: 'custom-file.yaml', + projectPattern: /[\s\S]*/, + groupPattern: /[\s\S]*/, + userPattern: /[\s\S]*/, + orgEnabled: false, + allowInherited: false, + relations: [], + schedule: undefined, + restrictUsersToGroup: false, + excludeRepos: [], + skipForkedRepos: false, + includeArchivedRepos: true, includeUsersWithoutSeat: false, }), ); @@ -192,6 +239,7 @@ describe('config', () => { schedule: undefined, restrictUsersToGroup: false, skipForkedRepos: false, + includeArchivedRepos: false, excludeRepos: ['foo/bar', 'quz/qux'], includeUsersWithoutSeat: false, }), @@ -235,6 +283,7 @@ describe('config', () => { allowInherited: false, relations: [], skipForkedRepos: false, + includeArchivedRepos: false, restrictUsersToGroup: false, excludeRepos: [], includeUsersWithoutSeat: false, diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index dd1f6d67d7..bf59bb0d36 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -49,6 +49,9 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { const skipForkedRepos: boolean = config.getOptionalBoolean('skipForkedRepos') ?? false; + + const includeArchivedRepos: boolean = + config.getOptionalBoolean('includeArchivedRepos') ?? false; const excludeRepos: string[] = config.getOptionalStringArray('excludeRepos') ?? []; @@ -78,6 +81,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { allowInherited, relations, skipForkedRepos, + includeArchivedRepos, excludeRepos, restrictUsersToGroup, includeUsersWithoutSeat,