paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 22 Packages Projects Releases Wiki Activity

Add support for includeArchivedRepos in catalog-backend-module-gitlab config, update docs

Browse Source 
Signed-off-by: Mkolaj Kaliszewski <kaliszewski.mikolaj@gmail.com>
This commit is contained in:
Mkolaj Kaliszewski
2024-10-18 11:23:59 +02:00
parent d336f929d8
commit 1b5fdd97a9
12 changed files with 274 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/silver-kings-sort.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-gitlab': minor
---
Extended the configuration with the includeArchivedRepos property, which allows including repositories when the project is archived.
docs/integrations/gitlab/discovery.md
+3
View File
@@ -150,6 +150,7 @@ catalog:
branch: main # Optional. Used to discover on a specific branch
fallbackBranch: master # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default
skipForkedRepos: false # Optional. If the project is a fork, skip repository
includeArchivedRepos: false # Optional. If project is archived, include repository
group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned
entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything
@@ -212,3 +213,5 @@ catalog:
If you don't want create location object if file with component definition do not exists in project, you can set the `skipReposWithoutExactFileMatch` option. That can reduce count of request to gitlab with 404 status code.
If you don't want to create location object if the project is a fork, you can set the `skipForkedRepos` option.
If you want to create location object if the project is archived, you can set the `includeArchivedRepos` option.
plugins/catalog-backend-module-gitlab/config.d.ts
Vendored
+4
View File
@@ -63,6 +63,10 @@ export interface Config {
* (Optional) Skip forked repository
*/
skipForkedRepos?: boolean;
/**
* (Optional) Include archived repository
*/
includeArchivedRepos?: boolean;
/**
* (Optional) A list of strings containing the paths of the repositories to skip
* Should be in the format group/subgroup/repo, with no leading or trailing slashes.
plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts
+5 -1
View File
@@ -39,6 +39,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
private readonly cache: CacheService;
private readonly skipReposWithoutExactFileMatch: boolean;
private readonly skipForkedRepos: boolean;
private readonly includeArchivedRepos: boolean;
static fromConfig(
config: Config,
@@ -46,6 +47,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
logger: LoggerService;
skipReposWithoutExactFileMatch?: boolean;
skipForkedRepos?: boolean;
includeArchivedRepos?: boolean;
},
): GitLabDiscoveryProcessor {
const integrations = ScmIntegrations.fromConfig(config);
@@ -65,6 +67,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
logger: LoggerService;
skipReposWithoutExactFileMatch?: boolean;
skipForkedRepos?: boolean;
includeArchivedRepos?: boolean;
}) {
this.integrations = options.integrations;
this.cache = options.pluginCache;
@@ -72,6 +75,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
this.skipReposWithoutExactFileMatch =
options.skipReposWithoutExactFileMatch || false;
this.skipForkedRepos = options.skipForkedRepos || false;
this.includeArchivedRepos = options.includeArchivedRepos || false;
}
getProcessorName(): string {
@@ -105,12 +109,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
const lastActivity = (await this.cache.get(this.getCacheKey())) as string;
const opts = {
archived: false,
group,
page: 1,
// We check for the existence of lastActivity and only set it if it's present to ensure
// that the options doesn't include the key so that the API doesn't receive an empty query parameter.
...(lastActivity && { last_activity_after: lastActivity }),
...(!this.includeArchivedRepos && { archived: false }),
};
const projects = paginated(options => client.listProjects(options), opts);
plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts
+26 -9
View File
@@ -37,9 +37,18 @@ const httpHandlers = [
* Project REST endpoint mocks
*/
// fetch all projects in an instance
rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => {
return res(ctx.set('x-next-page', ''), ctx.json(all_projects_response));
// fetch all projects in an instance handling archived ones
rest.get(`${apiBaseUrl}/projects`, (req, res, ctx) => {
const archived = req.url.searchParams.get('archived');
return res(
ctx.set('x-next-page', ''),
ctx.json(
all_projects_response.filter(p =>
archived === 'false' ? !p.archived : true,
),
),
);
}),
rest.get(`${apiBaseUrl}/projects/42`, (_, res, ctx) => {
@@ -136,9 +145,13 @@ const httpGroupFindByIdDynamic = all_groups_response.map(group => {
const httpGroupListDescendantProjectsById = all_groups_response.map(group => {
return rest.get(
`${apiBaseUrl}/groups/${group.id}/projects`,
(_, res, ctx) => {
const projectsInGroup = all_projects_response.filter(p =>
p.path_with_namespace?.includes(group.name),
(req, res, ctx) => {
const archived = req.url.searchParams.get('archived');
const projectsInGroup = all_projects_response.filter(
p =>
p.path_with_namespace?.includes(group.name) &&
(archived === 'false' ? !p.archived : true),
);
return res(ctx.json(projectsInGroup));
@@ -149,9 +162,13 @@ const httpGroupListDescendantProjectsById = all_groups_response.map(group => {
const httpGroupListDescendantProjectsByName = all_groups_response.map(group => {
return rest.get(
`${apiBaseUrl}/groups/${group.name}/projects`,
(_, res, ctx) => {
const projectsInGroup = all_projects_response.filter(p =>
p.path_with_namespace?.includes(group.name),
(req, res, ctx) => {
const archived = req.url.searchParams.get('archived');
const projectsInGroup = all_projects_response.filter(
p =>
p.path_with_namespace?.includes(group.name) &&
(archived === 'false' ? !p.archived : true),
);
return res(ctx.json(projectsInGroup));
plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts
+118 -44
View File
@@ -299,6 +299,33 @@ export const config_single_integration_skip_forks: MockObject = {
},
};
export const config_single_integration_include_archived: MockObject = {
integrations: {
gitlab: [
{
host: 'example.com',
apiBaseUrl: 'https://example.com/api/v4',
token: '1234',
},
],
},
catalog: {
providers: {
gitlab: {
'test-id': {
host: 'example.com',
group: 'group1',
includeArchivedRepos: true,
schedule: {
frequency: 'PT30M',
timeout: 'PT3M',
},
},
},
},
},
};
export const config_single_integration_exclude_repos: MockObject = {
integrations: {
gitlab: [
@@ -856,6 +883,18 @@ export const all_projects_response: GitLabProject[] = [
web_url: 'https://example.com/group1/test-repo7',
path_with_namespace: 'group1/test-repo7',
},
// archived project
{
id: 8,
description: 'Project Eight Description',
name: 'test-repo8-archived',
default_branch: 'main',
path: 'test-repo8-archived',
archived: true,
last_activity_at: new Date().toString(),
web_url: 'https://example.com/group1/test-repo8-archived',
path_with_namespace: 'group1/test-repo8-archived',
},
];
export const all_users_response: GitLabUser[] = [
@@ -1553,7 +1592,7 @@ export const push_modif_event: EventParams = {
// includes only projects that have a default branch (for when the branch and fallback branch were not set in the config)
export const expected_location_entities_default_branch: MockObject[] =
all_projects_response
.filter(project => project.default_branch)
.filter(project => project.default_branch && !project.archived)
.map(project => {
const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${project.default_branch}/catalog-info.yaml`;
@@ -1583,63 +1622,98 @@ export const expected_location_entities_default_branch: MockObject[] =
// includes every GitLab project that has a default branch and the fallback declared in the config
export const expected_location_entities_fallback_branch: MockObject[] =
all_projects_response.map(project => {
const branch = project.default_branch || 'main';
const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${branch}/catalog-info.yaml`;
all_projects_response
.filter(project => !project.archived)
.map(project => {
const branch = project.default_branch || 'main';
const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${branch}/catalog-info.yaml`;
return {
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${targetUrl}`,
'backstage.io/managed-by-origin-location': `url:${targetUrl}`,
return {
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${targetUrl}`,
'backstage.io/managed-by-origin-location': `url:${targetUrl}`,
},
name: locationSpecToMetadataName({
target: targetUrl,
type: 'url',
}),
},
name: locationSpecToMetadataName({
spec: {
presence: 'optional',
target: targetUrl,
type: 'url',
}),
},
},
spec: {
presence: 'optional',
target: targetUrl,
type: 'url',
},
},
locationKey: 'GitlabDiscoveryEntityProvider:test-id',
};
});
locationKey: 'GitlabDiscoveryEntityProvider:test-id',
};
});
// includes ONLY the projects with the branch declared in the config
export const expected_location_entities_specific_branch: MockObject[] =
all_projects_response.map(project => {
const branch = 'develop';
const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${branch}/catalog-info.yaml`;
all_projects_response
.filter(project => !project.archived)
.map(project => {
const branch = 'develop';
const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${branch}/catalog-info.yaml`;
return {
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${targetUrl}`,
'backstage.io/managed-by-origin-location': `url:${targetUrl}`,
return {
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${targetUrl}`,
'backstage.io/managed-by-origin-location': `url:${targetUrl}`,
},
name: locationSpecToMetadataName({
target: targetUrl,
type: 'url',
}),
},
name: locationSpecToMetadataName({
spec: {
presence: 'optional',
target: targetUrl,
type: 'url',
}),
},
},
spec: {
presence: 'optional',
target: targetUrl,
type: 'url',
locationKey: 'GitlabDiscoveryEntityProvider:test-id',
};
});
// includes archived and not archived projects
export const expected_location_entities_including_archived: MockObject[] =
all_projects_response
.filter(project => project.default_branch)
.map(project => {
const targetUrl = `https://example.com/${project.path_with_namespace}/-/blob/${project.default_branch}/catalog-info.yaml`;
return {
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${targetUrl}`,
'backstage.io/managed-by-origin-location': `url:${targetUrl}`,
},
name: locationSpecToMetadataName({
target: targetUrl,
type: 'url',
}),
},
spec: {
presence: 'optional',
target: targetUrl,
type: 'url',
},
},
},
locationKey: 'GitlabDiscoveryEntityProvider:test-id',
};
});
locationKey: 'GitlabDiscoveryEntityProvider:test-id',
};
});
export const expected_added_location_entities: MockObject[] = added_commits.map(
commit => {
plugins/catalog-backend-module-gitlab/src/lib/client.test.ts
+26
View File
@@ -143,6 +143,31 @@ describe('GitLabClient', () => {
expect(allItems).toHaveLength(expectedProjects.length);
});
it('should get not archived projects', async () => {
const client = new GitLabClient({
config: readGitLabIntegrationConfig(
new ConfigReader(mock.config_self_managed),
),
logger: mockServices.logger.mock(),
});
const archivedProjectsGen = paginated(
options => client.listProjects(options),
{ archived: false },
);
const expectedProjects = mock.all_projects_response.filter(
project => !project.archived,
);
const allItems = [];
for await (const item of archivedProjectsGen) {
allItems.push(item);
}
expect(allItems).toHaveLength(expectedProjects.length);
});
it('should get all projects for an instance', async () => {
const client = new GitLabClient({
config: readGitLabIntegrationConfig(
@@ -159,6 +184,7 @@ describe('GitLabClient', () => {
for await (const project of instanceProjects) {
allProjects.push(project);
}
expect(allProjects).toHaveLength(mock.all_projects_response.length);
});
});
plugins/catalog-backend-module-gitlab/src/lib/types.ts
+4
View File
@@ -232,6 +232,10 @@ export type GitlabProviderConfig = {
* If the project is a fork, skip repository
*/
skipForkedRepos?: boolean;
/**
* If the project is archived, include repository
*/
includeArchivedRepos?: boolean;
/**
* List of repositories to exclude from discovery, should be the full path to the repository, e.g. `group/project`
* Paths should not start or end with a slash.
plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts
+29
View File
@@ -232,6 +232,35 @@ describe('GitlabDiscoveryEntityProvider - refresh', () => {
});
});
it('should include archived projects', async () => {
const config = new ConfigReader(
mock.config_single_integration_include_archived,
);
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
await provider.connect(entityProviderConnection);
await provider.refresh(logger);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: mock.expected_location_entities_including_archived.filter(
entity =>
!entity.entity.metadata.annotations[
'backstage.io/managed-by-location'
].includes('awesome'),
),
});
});
it('should filter repositories that are excluded', async () => {
const config = new ConfigReader(
mock.config_single_integration_exclude_repos,
plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts
+1 -1
View File
@@ -210,10 +210,10 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
const projects = paginated<GitLabProject>(
options => this.gitLabClient.listProjects(options),
{
archived: false,
group: this.config.group,
page: 1,
per_page: 50,
...(!this.config.includeArchivedRepos && { archived: false }),
},
);
plugins/catalog-backend-module-gitlab/src/providers/config.test.ts
+49
View File
@@ -61,6 +61,7 @@ describe('config', () => {
relations: [],
schedule: undefined,
skipForkedRepos: false,
includeArchivedRepos: false,
excludeRepos: [],
restrictUsersToGroup: false,
includeUsersWithoutSeat: false,
@@ -104,6 +105,7 @@ describe('config', () => {
relations: [],
schedule: undefined,
skipForkedRepos: false,
includeArchivedRepos: false,
excludeRepos: [],
restrictUsersToGroup: false,
includeUsersWithoutSeat: true,
@@ -149,6 +151,51 @@ describe('config', () => {
restrictUsersToGroup: false,
excludeRepos: [],
skipForkedRepos: true,
includeArchivedRepos: false,
includeUsersWithoutSeat: false,
}),
);
});
it('valid config with includeArchivedRepos', () => {
const config = new ConfigReader({
catalog: {
providers: {
gitlab: {
test: {
group: 'group',
host: 'host',
branch: 'not-master',
fallbackBranch: 'main',
entityFilename: 'custom-file.yaml',
includeArchivedRepos: true,
},
},
},
},
});
const result = readGitlabConfigs(config);
expect(result).toHaveLength(1);
result.forEach(r =>
expect(r).toStrictEqual({
id: 'test',
group: 'group',
branch: 'not-master',
fallbackBranch: 'main',
host: 'host',
catalogFile: 'custom-file.yaml',
projectPattern: /[\s\S]*/,
groupPattern: /[\s\S]*/,
userPattern: /[\s\S]*/,
orgEnabled: false,
allowInherited: false,
relations: [],
schedule: undefined,
restrictUsersToGroup: false,
excludeRepos: [],
skipForkedRepos: false,
includeArchivedRepos: true,
includeUsersWithoutSeat: false,
}),
);
@@ -192,6 +239,7 @@ describe('config', () => {
schedule: undefined,
restrictUsersToGroup: false,
skipForkedRepos: false,
includeArchivedRepos: false,
excludeRepos: ['foo/bar', 'quz/qux'],
includeUsersWithoutSeat: false,
}),
@@ -235,6 +283,7 @@ describe('config', () => {
allowInherited: false,
relations: [],
skipForkedRepos: false,
includeArchivedRepos: false,
restrictUsersToGroup: false,
excludeRepos: [],
includeUsersWithoutSeat: false,
plugins/catalog-backend-module-gitlab/src/providers/config.ts
+4
View File
@@ -49,6 +49,9 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
const skipForkedRepos: boolean =
config.getOptionalBoolean('skipForkedRepos') ?? false;
const includeArchivedRepos: boolean =
config.getOptionalBoolean('includeArchivedRepos') ?? false;
const excludeRepos: string[] =
config.getOptionalStringArray('excludeRepos') ?? [];
@@ -78,6 +81,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
allowInherited,
relations,
skipForkedRepos,
includeArchivedRepos,
excludeRepos,
restrictUsersToGroup,
includeUsersWithoutSeat,