paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

Merge pull request #16502 from Andy2003/feature/13587-gitlab-discovery-branch-handling

Browse Source 
[GitlabDiscoveryEntityProvider] add config to distinguish between `branch` and `fallbackBranch`
This commit is contained in:
Johan Haals
2023-03-20 14:45:58 +01:00
committed by GitHub
parent f11f42b8e1 c11f33b842
commit 2d81a9122f
8 changed files with 148 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/fresh-schools-fly.md
+11
View File
@@ -0,0 +1,11 @@
---
'@backstage/plugin-catalog-backend-module-gitlab': minor
---
**BREAKING**: The configuration of the `GitlabDiscoveryEntityProvider` has changed as follows:
- The configuration key `branch` is now used to define the branch from which the catalog-info should be discovered.
- The old configuration key `branch` is now called `fallbackBranch`. This value specifies which branch should be used
if no default branch is defined on the project itself.
To migrate to the new configuration value, rename `branch` to `fallbackBranch`
docs/integrations/gitlab/discovery.md
+1
View File
@@ -21,6 +21,7 @@ catalog:
gitlab:
yourProviderId:
host: gitlab-host # Identifies one of the hosts set up in the integrations
branch: main # Optional. Used to discover on a specific branch
fallbackBranch: main # Optional. Fallback to be used if there is no default branch configured at the Gitlab repository. It is only used, if `branch` is undefined. Uses `master` as default
group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned
entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
plugins/catalog-backend-module-gitlab/src/lib/types.ts
+3 -2
View File
@@ -70,11 +70,12 @@ export type GitlabProviderConfig = {
group: string;
id: string;
/**
* @deprecated use `fallbackBranch` instead
* The name of the branch to be used, to discover catalog files.
*/
branch?: string;
/**
* If there is no default branch defined at the project, this fallback is used to discover catalog files.
* If no `branch` is configured and there is no default branch defined at the project as well, this fallback is used
* to discover catalog files.
* Defaults to: `master`
*/
fallbackBranch: string;
plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts
+107
View File
@@ -449,4 +449,111 @@ describe('GitlabDiscoveryEntityProvider', () => {
'GitlabDiscoveryEntityProvider:test-id',
);
});
it('should filter found projects based on the branch', async () => {
const config = new ConfigReader({
integrations: {
gitlab: [
{
host: 'test-gitlab',
apiBaseUrl: 'https://api.gitlab.example/api/v4',
token: '1234',
},
],
},
catalog: {
providers: {
gitlab: {
'test-id': {
host: 'test-gitlab',
branch: 'test',
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
server.use(
rest.get(
`https://api.gitlab.example/api/v4/projects`,
(_req, res, ctx) => {
const response = [
{
id: 123,
default_branch: 'master',
archived: false,
last_activity_at: new Date().toString(),
web_url: 'https://api.gitlab.example/test-group/test-repo',
path_with_namespace: 'test-group/test-repo',
},
{
id: 124,
default_branch: 'master',
archived: false,
last_activity_at: new Date().toString(),
web_url: 'https://api.gitlab.example/john/example',
path_with_namespace: 'john/example',
},
];
return res(ctx.json(response));
},
),
rest.head(
'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
(_, res, ctx) => {
return res(ctx.status(404, 'Not Found'));
},
),
rest.head(
'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml',
(req, res, ctx) => {
if (req.url.searchParams.get('ref') === 'test') {
return res(ctx.status(200));
}
return res(ctx.status(404, 'Not Found'));
},
),
);
await provider.connect(entityProviderConnection);
await provider.refresh(logger);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml',
'backstage.io/managed-by-origin-location':
'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml',
},
name: 'generated-232185d858fee049986d202c10316d634e76a3d1',
},
spec: {
presence: 'optional',
target:
'https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml',
type: 'url',
},
},
locationKey: 'GitlabDiscoveryEntityProvider:test-id',
},
],
});
});
});
plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts
+9 -6
View File
@@ -61,10 +61,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
throw new Error('Either schedule or scheduler must be provided.');
}
const providerConfigs = readGitlabConfigs(
config,
options.logger.child({ target: 'GitlabDiscoveryEntityProvider' }),
);
const providerConfigs = readGitlabConfigs(config);
const integrations = ScmIntegrations.fromConfig(config).gitlab;
const providers: GitlabDiscoveryEntityProvider[] = [];
@@ -181,6 +178,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
}
if (
!this.config.branch &&
this.config.fallbackBranch === '*' &&
project.default_branch === undefined
) {
@@ -188,7 +186,9 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
}
const project_branch =
project.default_branch ?? this.config.fallbackBranch;
this.config.branch ??
project.default_branch ??
this.config.fallbackBranch;
const projectHasFile: boolean = await client.hasFile(
project.path_with_namespace ?? '',
@@ -211,7 +211,10 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
}
private createLocationSpec(project: GitLabProject): LocationSpec {
const project_branch = project.default_branch ?? this.config.fallbackBranch;
const project_branch =
this.config.branch ??
project.default_branch ??
this.config.fallbackBranch;
return {
type: 'url',
target: `${project.web_url}/-/blob/${project_branch}/${this.config.catalogFile}`,
plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts
+1 -4
View File
@@ -72,10 +72,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
throw new Error('Either schedule or scheduler must be provided.');
}
const providerConfigs = readGitlabConfigs(
config,
options.logger.child({ target: 'GitlabOrgDiscoveryEntityProvider' }),
);
const providerConfigs = readGitlabConfigs(config);
const integrations = ScmIntegrations.fromConfig(config).gitlab;
const providers: GitlabOrgDiscoveryEntityProvider[] = [];
plugins/catalog-backend-module-gitlab/src/providers/config.test.ts
+11 -10
View File
@@ -17,9 +17,6 @@
import { ConfigReader } from '@backstage/config';
import { Duration } from 'luxon';
import { readGitlabConfigs } from './config';
import { getVoidLogger } from '@backstage/backend-common';
const logger = getVoidLogger();
describe('config', () => {
it('empty gitlab config', () => {
@@ -29,7 +26,7 @@ describe('config', () => {
},
});
const result = readGitlabConfigs(config, logger);
const result = readGitlabConfigs(config);
expect(result).toHaveLength(0);
});
@@ -47,12 +44,13 @@ describe('config', () => {
},
});
const result = readGitlabConfigs(config, logger);
const result = readGitlabConfigs(config);
expect(result).toHaveLength(1);
result.forEach(r =>
expect(r).toStrictEqual({
id: 'test',
group: 'group',
branch: undefined,
fallbackBranch: 'master',
host: 'host',
catalogFile: 'catalog-info.yaml',
@@ -74,6 +72,7 @@ describe('config', () => {
group: 'group',
host: 'host',
branch: 'not-master',
fallbackBranch: 'main',
entityFilename: 'custom-file.yaml',
},
},
@@ -81,13 +80,14 @@ describe('config', () => {
},
});
const result = readGitlabConfigs(config, logger);
const result = readGitlabConfigs(config);
expect(result).toHaveLength(1);
result.forEach(r =>
expect(r).toStrictEqual({
id: 'test',
group: 'group',
fallbackBranch: 'not-master',
branch: 'not-master',
fallbackBranch: 'main',
host: 'host',
catalogFile: 'custom-file.yaml',
projectPattern: /[\s\S]*/,
@@ -119,12 +119,13 @@ describe('config', () => {
},
});
const result = readGitlabConfigs(config, logger);
const result = readGitlabConfigs(config);
expect(result).toHaveLength(1);
result.forEach(r =>
expect(r).toStrictEqual({
id: 'test',
group: 'group',
branch: undefined,
fallbackBranch: 'master',
host: 'host',
catalogFile: 'catalog-info.yaml',
@@ -158,7 +159,7 @@ describe('config', () => {
},
});
expect(() => readGitlabConfigs(config, logger)).toThrow(
expect(() => readGitlabConfigs(config)).toThrow(
"Missing required config value at 'catalog.providers.gitlab.test.host'",
);
});
@@ -177,7 +178,7 @@ describe('config', () => {
},
});
const result = readGitlabConfigs(config, logger);
const result = readGitlabConfigs(config);
expect(result).toHaveLength(1);
expect(result[0].group).toEqual('');
});
plugins/catalog-backend-module-gitlab/src/providers/config.ts
+5 -28
View File
@@ -17,7 +17,6 @@
import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
import { GitlabProviderConfig } from '../lib';
import { Logger } from 'winston';
/**
* Extracts the gitlab config from a config object
@@ -26,25 +25,12 @@ import { Logger } from 'winston';
*
* @param id - The provider key
* @param config - The config object to extract from
* @param logger - The logger
*/
function readGitlabConfig(
id: string,
config: Config,
logger: Logger,
): GitlabProviderConfig {
function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
const group = config.getOptionalString('group') ?? '';
const host = config.getString('host');
const branch = config.getOptionalString('branch');
if (branch) {
logger.warn(
'The configuration key `branch` has been deprecated in favor of the configuration key `fallbackBranch`.',
);
}
const fallbackBranch =
config.getOptionalString('fallbackBranch') ?? branch ?? 'master';
const fallbackBranch = config.getOptionalString('fallbackBranch') ?? 'master';
const catalogFile =
config.getOptionalString('entityFilename') ?? 'catalog-info.yaml';
const projectPattern = new RegExp(
@@ -65,6 +51,7 @@ function readGitlabConfig(
return {
id,
group,
branch,
fallbackBranch,
host,
catalogFile,
@@ -82,12 +69,8 @@ function readGitlabConfig(
* @public
*
* @param config - The config object to extract from
* @param logger - The logger
*/
export function readGitlabConfigs(
config: Config,
logger: Logger,
): GitlabProviderConfig[] {
export function readGitlabConfigs(config: Config): GitlabProviderConfig[] {
const configs: GitlabProviderConfig[] = [];
const providerConfigs = config.getOptionalConfig('catalog.providers.gitlab');
@@ -97,13 +80,7 @@ export function readGitlabConfigs(
}
for (const id of providerConfigs.keys()) {
configs.push(
readGitlabConfig(
id,
providerConfigs.getConfig(id),
logger.child({ target: id }),
),
);
configs.push(readGitlabConfig(id, providerConfigs.getConfig(id)));
}
return configs;