diff --git a/.changeset/lazy-zoos-move.md b/.changeset/lazy-zoos-move.md new file mode 100644 index 0000000000..1e94ec798f --- /dev/null +++ b/.changeset/lazy-zoos-move.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Add a new provider `GitlabDiscoveryEntityProvider` as replacement for `GitlabDiscoveryProcessor` diff --git a/.changeset/perfect-mangos-allow.md b/.changeset/perfect-mangos-allow.md new file mode 100644 index 0000000000..0fe75e009c --- /dev/null +++ b/.changeset/perfect-mangos-allow.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add support to custom rules for `GitlabDiscoveryEntityProvider` diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 3715b89920..986c2aec15 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -6,14 +6,58 @@ sidebar_label: Discovery description: Automatically discovering catalog entities from repositories in GitLab --- -The GitLab integration has a special discovery processor for discovering catalog -entities from GitLab. The processor will crawl the GitLab instance and register -entities matching the configured path. This can be useful as an alternative to +The GitLab integration has a special entity provider for discovering catalog +entities from GitLab. The entity provider will crawl the GitLab instance and register +entities matching the configured paths. This can be useful as an alternative to static locations or manually adding things to the catalog. -To use the discovery processor, you'll need a GitLab integration -[set up](locations.md) with a `token`. Then you can add a location target to the -catalog configuration: +To use the discovery provider, you'll need a GitLab integration +[set up](locations.md) with a `token`. Then you can add a provider config per group +to the catalog configuration: + +```yaml +catalog: + providers: + gitlab: + yourProviderId: + host: gitlab-host # Identifies one of the hosts set up in the integrations + branch: main # Optional. Uses `master` as default + group: example-group # Group and subgroup (if needed) to look for repositories + entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` + rules: + - repository: example-repo + allow: [Component, System, Location, Template] +``` + +As this provider is not one of the default providers, you will first need to install +the gitlab catalog plugin: + +```bash +# From the Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-gitlab +``` + +Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`: + +```ts +/* packages/backend/src/plugins/catalog.ts */ + +import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-aws'; + +const builder = await CatalogBuilder.create(env); +/** ... other processors and/or providers ... */ +builder.addEntityProvider( + ...GitlabDiscoveryEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + }), +); +``` + +## Alternative processor ```yaml catalog: @@ -22,6 +66,9 @@ catalog: target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml ``` +As alternative to the entity provider `GitlabDiscoveryEntityProvider` +you can still use the `GitLabDiscoveryProcessor`. + Note the `gitlab-discovery` type, as this is not a regular `url` processor. The target is composed of three parts: diff --git a/plugins/catalog-backend-module-gitlab/config.d.ts b/plugins/catalog-backend-module-gitlab/config.d.ts new file mode 100644 index 0000000000..11c6830e63 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/config.d.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + catalog?: { + /** + * List of provider-specific options and attributes + */ + providers?: { + /** + * GitlabDiscoveryEntityProvider configuration + * + * Uses "default" as default id for the single config variant. + */ + gitlab?: Record< + string, + { + /** + * (Required) Gitlab's host name. + * @visibility backend + */ + host: string; + /** + * (Required) Gitlab's group[/subgroup] where the discovery is done. + * @visibility backend + */ + group: string; + /** + * (Optional) Default branch to read the catalog-info.yaml file. + * If not set, 'master' will be used. + * @visibility backend + */ + branch?: string; + /** + * (Optional) The name used for the catalog file. + * If not set, 'catalog-info.yaml' will be used. + * @visibility backend + */ + entityFilename?: string; + } + >; + }; + }; +} diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index d7909bf3b3..74615e83fe 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -35,6 +35,7 @@ "dependencies": { "@backstage/backend-common": "^0.14.0-next.2", "@backstage/catalog-model": "^1.0.3-next.0", + "@backstage/backend-tasks": "^0.3.1", "@backstage/config": "^1.0.1", "@backstage/errors": "^1.0.0", "@backstage/integration": "^1.2.1-next.2", @@ -43,12 +44,14 @@ "lodash": "^4.17.21", "msw": "^0.42.0", "node-fetch": "^2.6.7", - "winston": "^3.2.1" + "winston": "^3.2.1", + "uuid": "^8.0.0" }, "devDependencies": { - "@backstage/backend-test-utils": "^0.1.25-next.2", - "@backstage/cli": "^0.17.2-next.2", - "@types/lodash": "^4.14.151" + "@backstage/backend-test-utils": "^0.1.25-next.1", + "@backstage/cli": "^0.17.2-next.1", + "@types/lodash": "^4.14.151", + "@types/uuid": "^8.0.0" }, "files": [ "dist" diff --git a/plugins/catalog-backend-module-gitlab/src/index.ts b/plugins/catalog-backend-module-gitlab/src/index.ts index e2a08509fb..3182210c81 100644 --- a/plugins/catalog-backend-module-gitlab/src/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/index.ts @@ -21,3 +21,4 @@ */ export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; +export { GitlabDiscoveryEntityProvider } from './providers'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index e50ad34734..132dd0bc34 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -103,6 +103,21 @@ function setupFakeInstanceProjectsEndpoint( ); } +function setupFakeHasFileEndpoint(srv: SetupServerApi, apiBaseUrl: string) { + srv.use( + rest.head( + `${apiBaseUrl}/projects/group%2Frepo/repository/files/catalog-info.yaml`, + (req, res, ctx) => { + const branch = req.url.searchParams.get('ref'); + if (branch === 'master') { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ), + ); +} + describe('GitLabClient', () => { describe('isSelfManaged', () => { it('returns true if self managed instance', () => { @@ -266,3 +281,33 @@ describe('paginated', () => { expect(allItems).toHaveLength(4); }); }); + +describe('hasFile', () => { + let client: GitLabClient; + + beforeEach(() => { + setupFakeHasFileEndpoint(server, MOCK_CONFIG.apiBaseUrl); + client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + }); + + it('should not find catalog file', async () => { + const hasFile = await client.hasFile( + 'group/repo', + 'master', + 'catalog-info.yaml', + ); + expect(hasFile).toBe(true); + }); + + it('should find catalog file', async () => { + const hasFile = await client.hasFile( + 'group/repo', + 'unknown', + 'catalog-info.yaml', + ); + expect(hasFile).toBe(false); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 27214f3be8..25384f3dd6 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -63,6 +63,14 @@ export class GitLabClient { return this.pagedRequest(`/projects`, options); } + /** + * Checks if the catalog file is present in the repository or not. + * + * @param projectPath The path to the project + * @param branch The branch used to injest entities to the catalog + * @param filePath The path to the catalog file + * @returns `true` if the file exists, `false` otherwise + */ async hasFile( projectPath: string, branch: string, diff --git a/plugins/catalog-backend-module-gitlab/src/lib/index.ts b/plugins/catalog-backend-module-gitlab/src/lib/index.ts index 1df3d2cb84..53fad07994 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/index.ts @@ -15,4 +15,9 @@ */ export { GitLabClient, paginated } from './client'; -export type { GitLabProject } from './types'; +export type { + GitLabProject, + GitlabProviderConfig, + GitlabGroupDescription, +} from './types'; +export { readGitlabConfigs } from '../providers/config'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index d41ed73aac..fa2350487f 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -14,10 +14,25 @@ * limitations under the License. */ +export type GitlabGroupDescription = { + id: number; + web_url: string; + projects: GitLabProject[]; +}; + export type GitLabProject = { id: number; default_branch?: string; archived: boolean; last_activity_at: string; web_url: string; + path_with_namespace: string; +}; + +export type GitlabProviderConfig = { + host: string; + group: string; + id: string; + branch: string; + catalogFile: string; }; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts new file mode 100644 index 0000000000..dc70772d3d --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -0,0 +1,230 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const logger = getVoidLogger(); + +const server = setupServer(); + +describe('GitlabDiscoveryEntityProvider', () => { + setupRequestMockHandlers(server); + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({}); + const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(0); + }); + + it('single simple discovery config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + }, + }, + }, + }); + const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + }); + + it('multiple discovery configs', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + 'second-test': { + host: 'test-gitlab', + group: 'second-group', + }, + }, + }, + }, + }); + const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(2); + expect(providers[0].getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + expect(providers[1].getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:second-test', + ); + }); + + it('apply full update on scheduled execution', async () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + + server.use( + rest.get( + `https://api.gitlab.example/api/v4/groups/test-group/projects`, + (_req, res, ctx) => { + const response = [ + { + id: 123, + default_branch: 'master', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://api.gitlab.example/test-group/test-repo', + path_with_namespace: 'test-group/test-repo', + }, + ]; + return res(ctx.json(response)); + }, + ), + rest.head( + 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', + (req, res, ctx) => { + if (req.url.searchParams.get('ref') === 'master') { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('GitlabDiscoveryEntityProvider:test-id:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://api.gitlab.example/test-group/test-repo/-/blob/master/catalog-info.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-cd37bf72a2fe92603f4255d9f49c6c1ead746a48', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }, + ]; + + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts new file mode 100644 index 0000000000..7a30aa83da --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -0,0 +1,190 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskRunner } from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import { LocationSpec } from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { + GitLabClient, + GitLabProject, + GitlabProviderConfig, + paginated, + readGitlabConfigs, +} from '../lib'; +import * as uuid from 'uuid'; +import { locationSpecToLocationEntity } from '@backstage/plugin-catalog-backend'; + +type Result = { + scanned: number; + matches: GitLabProject[]; +}; + +/** + * Extracts repositories out of an GitLab instance. + * @public + */ +export class GitlabDiscoveryEntityProvider implements EntityProvider { + private readonly config: GitlabProviderConfig; + private readonly integration: GitLabIntegration; + private readonly logger: Logger; + private readonly scheduleFn: () => Promise; + private connection?: EntityProviderConnection; + + static fromConfig( + config: Config, + options: { logger: Logger; schedule: TaskRunner }, + ): GitlabDiscoveryEntityProvider[] { + const providerConfigs = readGitlabConfigs(config); + const integrations = ScmIntegrations.fromConfig(config).gitlab; + const providers: GitlabDiscoveryEntityProvider[] = []; + + providerConfigs.forEach(providerConfig => { + const integration = integrations.byHost(providerConfig.host); + if (!integration) { + throw new Error( + `No gitlab integration found that matches host ${providerConfig.host}`, + ); + } + providers.push( + new GitlabDiscoveryEntityProvider({ + ...options, + config: providerConfig, + integration, + }), + ); + }); + return providers; + } + + private constructor(options: { + config: GitlabProviderConfig; + integration: GitLabIntegration; + logger: Logger; + schedule: TaskRunner; + }) { + this.config = options.config; + this.integration = options.integration; + this.logger = options.logger.child({ + target: this.getProviderName(), + }); + this.scheduleFn = this.createScheduleFn(options.schedule); + } + + getProviderName(): string { + return `GitlabDiscoveryEntityProvider:${this.config.id}`; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + await this.scheduleFn(); + } + + private createScheduleFn(schedule: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return schedule.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: GitlabDiscoveryEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + + try { + await this.refresh(logger); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } + + async refresh(logger: Logger): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const client = new GitLabClient({ + config: this.integration.config, + logger: logger, + }); + + const projects = paginated( + options => client.listProjects(options), + { + group: this.config.group, + page: 1, + per_page: 50, + }, + ); + + const res: Result = { + scanned: 0, + matches: [], + }; + + for await (const project of projects) { + res.scanned++; + + if (project.archived) { + continue; + } + + if (this.config.branch === '*' && project.default_branch === undefined) { + continue; + } + + const project_branch = project.default_branch ?? this.config.branch; + + const projectHasFile: boolean = await client.hasFile( + project.path_with_namespace, + project_branch, + this.config.catalogFile, + ); + if (projectHasFile) { + res.matches.push(project); + } + } + + const locations = res.matches.map(p => this.createLocationSpec(p)); + await this.connection.applyMutation({ + type: 'full', + entities: locations.map(location => ({ + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + })), + }); + } + + private createLocationSpec(project: GitLabProject): LocationSpec { + const project_branch = project.default_branch ?? this.config.branch; + return { + type: 'url', + target: `${project.web_url}/-/blob/${project_branch}/${this.config.catalogFile}`, + presence: 'optional', + }; + } +} diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts new file mode 100644 index 0000000000..d28930b151 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readGitlabConfigs } from './config'; + +describe('config', () => { + it('empty gitlab config', () => { + const config = new ConfigReader({ + catalog: { + providers: {}, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(0); + }); + + it('valid config with default optional params', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + group: 'group', + host: 'host', + }, + }, + }, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(1); + result.forEach(r => + expect(r).toStrictEqual({ + id: 'test', + group: 'group', + branch: 'master', + host: 'host', + catalogFile: 'catalog-info.yaml', + }), + ); + }); + + it('valid config with custom optional params', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + group: 'group', + host: 'host', + branch: 'not-master', + entityFilename: 'custom-file.yaml', + }, + }, + }, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(1); + result.forEach(r => + expect(r).toStrictEqual({ + id: 'test', + group: 'group', + branch: 'not-master', + host: 'host', + catalogFile: 'custom-file.yaml', + }), + ); + }); + + it('missing params', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + branch: 'not-master', + entityFilename: 'custom-file.yaml', + }, + }, + }, + }, + }); + + expect(() => readGitlabConfigs(config)).toThrow( + "Missing required config value at 'catalog.providers.gitlab.test.group'", + ); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts new file mode 100644 index 0000000000..8af3d576a7 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { GitlabProviderConfig } from '../lib/types'; + +function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { + const group = config.getString('group'); + const host = config.getString('host'); + const branch = config.getOptionalString('branch') ?? 'master'; + const catalogFile = + config.getOptionalString('entityFilename') ?? 'catalog-info.yaml'; + + return { + id, + group, + branch, + host, + catalogFile, + }; +} + +export function readGitlabConfigs(config: Config): GitlabProviderConfig[] { + const configs: GitlabProviderConfig[] = []; + + const providerConfigs = config.getOptionalConfig('catalog.providers.gitlab'); + + if (!providerConfigs) { + return configs; + } + + for (const id of providerConfigs.keys()) { + configs.push(readGitlabConfig(id, providerConfigs.getConfig(id))); + } + + return configs; +} diff --git a/plugins/catalog-backend-module-gitlab/src/providers/index.ts b/plugins/catalog-backend-module-gitlab/src/providers/index.ts new file mode 100644 index 0000000000..e7cb00a73f --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 587a0bd7d5..72eae22b11 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import path from 'path'; import { LocationSpec } from '../api'; +import { ScmIntegrations } from '@backstage/integration'; /** * Rules to apply to catalog entities. @@ -119,6 +120,18 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { rules.push(...locationRules); } + if (config.has('catalog.providers')) { + const providersConf = config.getConfig('catalog.providers'); + const providerList = providersConf.keys(); + providerList.forEach(provider => { + if (provider === 'gitlab') { + rules.push( + ...getGitlabRules(config, providersConf.getConfig(provider)), + ); + } + }); + } + return new DefaultCatalogRulesEnforcer(rules); } @@ -187,3 +200,35 @@ function resolveTarget(type: string, target: string): string { return path.resolve(target); } + +function getGitlabRules(initialConfig: Config, config: Config): CatalogRule[] { + return config.keys().flatMap(id => { + const gitlabConf = config.getConfig(id); + if (!gitlabConf.has('rules')) { + return []; + } + + const integrations = ScmIntegrations.fromConfig(initialConfig).gitlab; + const gitlabHost = gitlabConf.getString('host'); + const integration = integrations.byHost(gitlabHost); + if (!integration) { + throw new Error( + `No gitlab integration found that matches host ${gitlabHost}`, + ); + } + + const type = `url`; + const branch = gitlabConf.getOptionalString('branch') ?? 'master'; + const entityFilename = + gitlabConf.getOptionalString('entityFilename') ?? 'catalog-info.yaml'; + + return gitlabConf.getConfigArray('rules').map(ruleConf => { + const repoName = ruleConf.getString('repository'); + const target = `${integration.config.baseUrl}/${id}/${repoName}/-/blob/${branch}/${entityFilename}`; + return { + allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), + locations: [{ type, target }], + }; + }); + }); +}