From eea8126171135c356f7e6bf30f0434e9634a7f57 Mon Sep 17 00:00:00 2001 From: ivgo Date: Tue, 7 Jun 2022 08:24:40 +0200 Subject: [PATCH 1/6] Gitlab entity provider Signed-off-by: ivgo --- .changeset/lazy-zoos-move.md | 5 + .changeset/perfect-mangos-allow.md | 5 + docs/integrations/gitlab/discovery.md | 59 ++++- .../catalog-backend-module-gitlab/config.d.ts | 57 +++++ .../package.json | 11 +- .../src/index.ts | 1 + .../src/lib/client.test.ts | 45 ++++ .../src/lib/client.ts | 8 + .../src/lib/index.ts | 7 +- .../src/lib/types.ts | 15 ++ .../GitlabDiscoveryEntityProvider.test.ts | 230 ++++++++++++++++++ .../GitlabDiscoveryEntityProvider.ts | 190 +++++++++++++++ .../src/providers/config.test.ts | 106 ++++++++ .../src/providers/config.ts | 50 ++++ .../src/providers/index.ts | 17 ++ .../src/ingestion/CatalogRules.ts | 45 ++++ 16 files changed, 840 insertions(+), 11 deletions(-) create mode 100644 .changeset/lazy-zoos-move.md create mode 100644 .changeset/perfect-mangos-allow.md create mode 100644 plugins/catalog-backend-module-gitlab/config.d.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/providers/config.test.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/providers/config.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/providers/index.ts diff --git a/.changeset/lazy-zoos-move.md b/.changeset/lazy-zoos-move.md new file mode 100644 index 0000000000..1e94ec798f --- /dev/null +++ b/.changeset/lazy-zoos-move.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Add a new provider `GitlabDiscoveryEntityProvider` as replacement for `GitlabDiscoveryProcessor` diff --git a/.changeset/perfect-mangos-allow.md b/.changeset/perfect-mangos-allow.md new file mode 100644 index 0000000000..0fe75e009c --- /dev/null +++ b/.changeset/perfect-mangos-allow.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add support to custom rules for `GitlabDiscoveryEntityProvider` diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 3715b89920..986c2aec15 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -6,14 +6,58 @@ sidebar_label: Discovery description: Automatically discovering catalog entities from repositories in GitLab --- -The GitLab integration has a special discovery processor for discovering catalog -entities from GitLab. The processor will crawl the GitLab instance and register -entities matching the configured path. This can be useful as an alternative to +The GitLab integration has a special entity provider for discovering catalog +entities from GitLab. The entity provider will crawl the GitLab instance and register +entities matching the configured paths. This can be useful as an alternative to static locations or manually adding things to the catalog. -To use the discovery processor, you'll need a GitLab integration -[set up](locations.md) with a `token`. Then you can add a location target to the -catalog configuration: +To use the discovery provider, you'll need a GitLab integration +[set up](locations.md) with a `token`. Then you can add a provider config per group +to the catalog configuration: + +```yaml +catalog: + providers: + gitlab: + yourProviderId: + host: gitlab-host # Identifies one of the hosts set up in the integrations + branch: main # Optional. Uses `master` as default + group: example-group # Group and subgroup (if needed) to look for repositories + entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` + rules: + - repository: example-repo + allow: [Component, System, Location, Template] +``` + +As this provider is not one of the default providers, you will first need to install +the gitlab catalog plugin: + +```bash +# From the Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-gitlab +``` + +Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`: + +```ts +/* packages/backend/src/plugins/catalog.ts */ + +import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-aws'; + +const builder = await CatalogBuilder.create(env); +/** ... other processors and/or providers ... */ +builder.addEntityProvider( + ...GitlabDiscoveryEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + }), +); +``` + +## Alternative processor ```yaml catalog: @@ -22,6 +66,9 @@ catalog: target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml ``` +As alternative to the entity provider `GitlabDiscoveryEntityProvider` +you can still use the `GitLabDiscoveryProcessor`. + Note the `gitlab-discovery` type, as this is not a regular `url` processor. The target is composed of three parts: diff --git a/plugins/catalog-backend-module-gitlab/config.d.ts b/plugins/catalog-backend-module-gitlab/config.d.ts new file mode 100644 index 0000000000..11c6830e63 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/config.d.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + catalog?: { + /** + * List of provider-specific options and attributes + */ + providers?: { + /** + * GitlabDiscoveryEntityProvider configuration + * + * Uses "default" as default id for the single config variant. + */ + gitlab?: Record< + string, + { + /** + * (Required) Gitlab's host name. + * @visibility backend + */ + host: string; + /** + * (Required) Gitlab's group[/subgroup] where the discovery is done. + * @visibility backend + */ + group: string; + /** + * (Optional) Default branch to read the catalog-info.yaml file. + * If not set, 'master' will be used. + * @visibility backend + */ + branch?: string; + /** + * (Optional) The name used for the catalog file. + * If not set, 'catalog-info.yaml' will be used. + * @visibility backend + */ + entityFilename?: string; + } + >; + }; + }; +} diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index d7909bf3b3..74615e83fe 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -35,6 +35,7 @@ "dependencies": { "@backstage/backend-common": "^0.14.0-next.2", "@backstage/catalog-model": "^1.0.3-next.0", + "@backstage/backend-tasks": "^0.3.1", "@backstage/config": "^1.0.1", "@backstage/errors": "^1.0.0", "@backstage/integration": "^1.2.1-next.2", @@ -43,12 +44,14 @@ "lodash": "^4.17.21", "msw": "^0.42.0", "node-fetch": "^2.6.7", - "winston": "^3.2.1" + "winston": "^3.2.1", + "uuid": "^8.0.0" }, "devDependencies": { - "@backstage/backend-test-utils": "^0.1.25-next.2", - "@backstage/cli": "^0.17.2-next.2", - "@types/lodash": "^4.14.151" + "@backstage/backend-test-utils": "^0.1.25-next.1", + "@backstage/cli": "^0.17.2-next.1", + "@types/lodash": "^4.14.151", + "@types/uuid": "^8.0.0" }, "files": [ "dist" diff --git a/plugins/catalog-backend-module-gitlab/src/index.ts b/plugins/catalog-backend-module-gitlab/src/index.ts index e2a08509fb..3182210c81 100644 --- a/plugins/catalog-backend-module-gitlab/src/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/index.ts @@ -21,3 +21,4 @@ */ export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; +export { GitlabDiscoveryEntityProvider } from './providers'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index e50ad34734..132dd0bc34 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -103,6 +103,21 @@ function setupFakeInstanceProjectsEndpoint( ); } +function setupFakeHasFileEndpoint(srv: SetupServerApi, apiBaseUrl: string) { + srv.use( + rest.head( + `${apiBaseUrl}/projects/group%2Frepo/repository/files/catalog-info.yaml`, + (req, res, ctx) => { + const branch = req.url.searchParams.get('ref'); + if (branch === 'master') { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ), + ); +} + describe('GitLabClient', () => { describe('isSelfManaged', () => { it('returns true if self managed instance', () => { @@ -266,3 +281,33 @@ describe('paginated', () => { expect(allItems).toHaveLength(4); }); }); + +describe('hasFile', () => { + let client: GitLabClient; + + beforeEach(() => { + setupFakeHasFileEndpoint(server, MOCK_CONFIG.apiBaseUrl); + client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + }); + + it('should not find catalog file', async () => { + const hasFile = await client.hasFile( + 'group/repo', + 'master', + 'catalog-info.yaml', + ); + expect(hasFile).toBe(true); + }); + + it('should find catalog file', async () => { + const hasFile = await client.hasFile( + 'group/repo', + 'unknown', + 'catalog-info.yaml', + ); + expect(hasFile).toBe(false); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 27214f3be8..25384f3dd6 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -63,6 +63,14 @@ export class GitLabClient { return this.pagedRequest(`/projects`, options); } + /** + * Checks if the catalog file is present in the repository or not. + * + * @param projectPath The path to the project + * @param branch The branch used to injest entities to the catalog + * @param filePath The path to the catalog file + * @returns `true` if the file exists, `false` otherwise + */ async hasFile( projectPath: string, branch: string, diff --git a/plugins/catalog-backend-module-gitlab/src/lib/index.ts b/plugins/catalog-backend-module-gitlab/src/lib/index.ts index 1df3d2cb84..53fad07994 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/index.ts @@ -15,4 +15,9 @@ */ export { GitLabClient, paginated } from './client'; -export type { GitLabProject } from './types'; +export type { + GitLabProject, + GitlabProviderConfig, + GitlabGroupDescription, +} from './types'; +export { readGitlabConfigs } from '../providers/config'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index d41ed73aac..fa2350487f 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -14,10 +14,25 @@ * limitations under the License. */ +export type GitlabGroupDescription = { + id: number; + web_url: string; + projects: GitLabProject[]; +}; + export type GitLabProject = { id: number; default_branch?: string; archived: boolean; last_activity_at: string; web_url: string; + path_with_namespace: string; +}; + +export type GitlabProviderConfig = { + host: string; + group: string; + id: string; + branch: string; + catalogFile: string; }; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts new file mode 100644 index 0000000000..dc70772d3d --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -0,0 +1,230 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const logger = getVoidLogger(); + +const server = setupServer(); + +describe('GitlabDiscoveryEntityProvider', () => { + setupRequestMockHandlers(server); + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({}); + const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(0); + }); + + it('single simple discovery config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + }, + }, + }, + }); + const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + }); + + it('multiple discovery configs', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + 'second-test': { + host: 'test-gitlab', + group: 'second-group', + }, + }, + }, + }, + }); + const providers = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(2); + expect(providers[0].getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + expect(providers[1].getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:second-test', + ); + }); + + it('apply full update on scheduled execution', async () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabDiscoveryEntityProvider:test-id', + ); + + server.use( + rest.get( + `https://api.gitlab.example/api/v4/groups/test-group/projects`, + (_req, res, ctx) => { + const response = [ + { + id: 123, + default_branch: 'master', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://api.gitlab.example/test-group/test-repo', + path_with_namespace: 'test-group/test-repo', + }, + ]; + return res(ctx.json(response)); + }, + ), + rest.head( + 'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml', + (req, res, ctx) => { + if (req.url.searchParams.get('ref') === 'master') { + return res(ctx.status(200)); + } + return res(ctx.status(404, 'Not Found')); + }, + ), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('GitlabDiscoveryEntityProvider:test-id:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://api.gitlab.example/test-group/test-repo/-/blob/master/catalog-info.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-cd37bf72a2fe92603f4255d9f49c6c1ead746a48', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'GitlabDiscoveryEntityProvider:test-id', + }, + ]; + + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts new file mode 100644 index 0000000000..7a30aa83da --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -0,0 +1,190 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskRunner } from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import { LocationSpec } from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { + GitLabClient, + GitLabProject, + GitlabProviderConfig, + paginated, + readGitlabConfigs, +} from '../lib'; +import * as uuid from 'uuid'; +import { locationSpecToLocationEntity } from '@backstage/plugin-catalog-backend'; + +type Result = { + scanned: number; + matches: GitLabProject[]; +}; + +/** + * Extracts repositories out of an GitLab instance. + * @public + */ +export class GitlabDiscoveryEntityProvider implements EntityProvider { + private readonly config: GitlabProviderConfig; + private readonly integration: GitLabIntegration; + private readonly logger: Logger; + private readonly scheduleFn: () => Promise; + private connection?: EntityProviderConnection; + + static fromConfig( + config: Config, + options: { logger: Logger; schedule: TaskRunner }, + ): GitlabDiscoveryEntityProvider[] { + const providerConfigs = readGitlabConfigs(config); + const integrations = ScmIntegrations.fromConfig(config).gitlab; + const providers: GitlabDiscoveryEntityProvider[] = []; + + providerConfigs.forEach(providerConfig => { + const integration = integrations.byHost(providerConfig.host); + if (!integration) { + throw new Error( + `No gitlab integration found that matches host ${providerConfig.host}`, + ); + } + providers.push( + new GitlabDiscoveryEntityProvider({ + ...options, + config: providerConfig, + integration, + }), + ); + }); + return providers; + } + + private constructor(options: { + config: GitlabProviderConfig; + integration: GitLabIntegration; + logger: Logger; + schedule: TaskRunner; + }) { + this.config = options.config; + this.integration = options.integration; + this.logger = options.logger.child({ + target: this.getProviderName(), + }); + this.scheduleFn = this.createScheduleFn(options.schedule); + } + + getProviderName(): string { + return `GitlabDiscoveryEntityProvider:${this.config.id}`; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + await this.scheduleFn(); + } + + private createScheduleFn(schedule: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return schedule.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: GitlabDiscoveryEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + + try { + await this.refresh(logger); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } + + async refresh(logger: Logger): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const client = new GitLabClient({ + config: this.integration.config, + logger: logger, + }); + + const projects = paginated( + options => client.listProjects(options), + { + group: this.config.group, + page: 1, + per_page: 50, + }, + ); + + const res: Result = { + scanned: 0, + matches: [], + }; + + for await (const project of projects) { + res.scanned++; + + if (project.archived) { + continue; + } + + if (this.config.branch === '*' && project.default_branch === undefined) { + continue; + } + + const project_branch = project.default_branch ?? this.config.branch; + + const projectHasFile: boolean = await client.hasFile( + project.path_with_namespace, + project_branch, + this.config.catalogFile, + ); + if (projectHasFile) { + res.matches.push(project); + } + } + + const locations = res.matches.map(p => this.createLocationSpec(p)); + await this.connection.applyMutation({ + type: 'full', + entities: locations.map(location => ({ + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + })), + }); + } + + private createLocationSpec(project: GitLabProject): LocationSpec { + const project_branch = project.default_branch ?? this.config.branch; + return { + type: 'url', + target: `${project.web_url}/-/blob/${project_branch}/${this.config.catalogFile}`, + presence: 'optional', + }; + } +} diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts new file mode 100644 index 0000000000..d28930b151 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readGitlabConfigs } from './config'; + +describe('config', () => { + it('empty gitlab config', () => { + const config = new ConfigReader({ + catalog: { + providers: {}, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(0); + }); + + it('valid config with default optional params', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + group: 'group', + host: 'host', + }, + }, + }, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(1); + result.forEach(r => + expect(r).toStrictEqual({ + id: 'test', + group: 'group', + branch: 'master', + host: 'host', + catalogFile: 'catalog-info.yaml', + }), + ); + }); + + it('valid config with custom optional params', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + group: 'group', + host: 'host', + branch: 'not-master', + entityFilename: 'custom-file.yaml', + }, + }, + }, + }, + }); + + const result = readGitlabConfigs(config); + expect(result).toHaveLength(1); + result.forEach(r => + expect(r).toStrictEqual({ + id: 'test', + group: 'group', + branch: 'not-master', + host: 'host', + catalogFile: 'custom-file.yaml', + }), + ); + }); + + it('missing params', () => { + const config = new ConfigReader({ + catalog: { + providers: { + gitlab: { + test: { + branch: 'not-master', + entityFilename: 'custom-file.yaml', + }, + }, + }, + }, + }); + + expect(() => readGitlabConfigs(config)).toThrow( + "Missing required config value at 'catalog.providers.gitlab.test.group'", + ); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts new file mode 100644 index 0000000000..8af3d576a7 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { GitlabProviderConfig } from '../lib/types'; + +function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { + const group = config.getString('group'); + const host = config.getString('host'); + const branch = config.getOptionalString('branch') ?? 'master'; + const catalogFile = + config.getOptionalString('entityFilename') ?? 'catalog-info.yaml'; + + return { + id, + group, + branch, + host, + catalogFile, + }; +} + +export function readGitlabConfigs(config: Config): GitlabProviderConfig[] { + const configs: GitlabProviderConfig[] = []; + + const providerConfigs = config.getOptionalConfig('catalog.providers.gitlab'); + + if (!providerConfigs) { + return configs; + } + + for (const id of providerConfigs.keys()) { + configs.push(readGitlabConfig(id, providerConfigs.getConfig(id))); + } + + return configs; +} diff --git a/plugins/catalog-backend-module-gitlab/src/providers/index.ts b/plugins/catalog-backend-module-gitlab/src/providers/index.ts new file mode 100644 index 0000000000..e7cb00a73f --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 587a0bd7d5..72eae22b11 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import path from 'path'; import { LocationSpec } from '../api'; +import { ScmIntegrations } from '@backstage/integration'; /** * Rules to apply to catalog entities. @@ -119,6 +120,18 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { rules.push(...locationRules); } + if (config.has('catalog.providers')) { + const providersConf = config.getConfig('catalog.providers'); + const providerList = providersConf.keys(); + providerList.forEach(provider => { + if (provider === 'gitlab') { + rules.push( + ...getGitlabRules(config, providersConf.getConfig(provider)), + ); + } + }); + } + return new DefaultCatalogRulesEnforcer(rules); } @@ -187,3 +200,35 @@ function resolveTarget(type: string, target: string): string { return path.resolve(target); } + +function getGitlabRules(initialConfig: Config, config: Config): CatalogRule[] { + return config.keys().flatMap(id => { + const gitlabConf = config.getConfig(id); + if (!gitlabConf.has('rules')) { + return []; + } + + const integrations = ScmIntegrations.fromConfig(initialConfig).gitlab; + const gitlabHost = gitlabConf.getString('host'); + const integration = integrations.byHost(gitlabHost); + if (!integration) { + throw new Error( + `No gitlab integration found that matches host ${gitlabHost}`, + ); + } + + const type = `url`; + const branch = gitlabConf.getOptionalString('branch') ?? 'master'; + const entityFilename = + gitlabConf.getOptionalString('entityFilename') ?? 'catalog-info.yaml'; + + return gitlabConf.getConfigArray('rules').map(ruleConf => { + const repoName = ruleConf.getString('repository'); + const target = `${integration.config.baseUrl}/${id}/${repoName}/-/blob/${branch}/${entityFilename}`; + return { + allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), + locations: [{ type, target }], + }; + }); + }); +} From 459efebf87bdce2ae17e269608b6af7c227169a0 Mon Sep 17 00:00:00 2001 From: ivgo Date: Tue, 7 Jun 2022 08:48:49 +0200 Subject: [PATCH 2/6] Update api-report Signed-off-by: ivgo --- .../api-report.md | 21 ++++ .../src/providers/config.ts | 14 +++ yarn.lock | 100 ++++++++++++++++++ 3 files changed, 135 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index 63e6fe7ccb..1009da8209 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -6,8 +6,29 @@ import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; +import { EntityProvider } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { TaskRunner } from '@backstage/backend-tasks'; + +// @public +export class GitlabDiscoveryEntityProvider implements EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger; + schedule: TaskRunner; + }, + ): GitlabDiscoveryEntityProvider[]; + // (undocumented) + getProviderName(): string; + // (undocumented) + refresh(logger: Logger): Promise; +} // @public export class GitLabDiscoveryProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index 8af3d576a7..f7eb316997 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -17,6 +17,13 @@ import { Config } from '@backstage/config'; import { GitlabProviderConfig } from '../lib/types'; +/** + * Extracts the gitlab config from a config object + * + * @public + * + * @param config - The config object to extract from + */ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { const group = config.getString('group'); const host = config.getString('host'); @@ -33,6 +40,13 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { }; } +/** + * Extracts the gitlab config from a config object array + * + * @public + * + * @param config - The config object to extract from + */ export function readGitlabConfigs(config: Config): GitlabProviderConfig[] { const configs: GitlabProviderConfig[] = []; diff --git a/yarn.lock b/yarn.lock index c7795b8004..fb4b8f867c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1465,6 +1465,78 @@ "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" +"@backstage/backend-common@^0.13.3": + version "0.13.5" + resolved "https://registry.npmjs.org/@backstage/backend-common/-/backend-common-0.13.5.tgz#9ff55e1da2a9cf65d0c4782cb360f9b6abd0754c" + integrity sha512-37kLw1BolfstantREuawbyRKXD+xKHUta3A63j0868JZPxRSCfiysBzF3uAorH/3WIh0p2lswutmunbVQFsXDA== + dependencies: + "@backstage/cli-common" "^0.1.9" + "@backstage/config" "^1.0.1" + "@backstage/config-loader" "^1.1.1" + "@backstage/errors" "^1.0.0" + "@backstage/integration" "^1.2.0" + "@backstage/types" "^1.0.0" + "@google-cloud/storage" "^5.8.0" + "@keyv/redis" "^2.2.3" + "@manypkg/get-packages" "^1.1.3" + "@octokit/rest" "^18.5.3" + "@types/cors" "^2.8.6" + "@types/dockerode" "^3.3.0" + "@types/express" "^4.17.6" + "@types/luxon" "^2.0.4" + "@types/webpack-env" "^1.15.2" + archiver "^5.0.2" + aws-sdk "^2.840.0" + base64-stream "^1.0.0" + compression "^1.7.4" + concat-stream "^2.0.0" + cors "^2.8.5" + dockerode "^3.3.1" + express "^4.17.1" + express-promise-router "^4.1.0" + fs-extra "10.1.0" + git-url-parse "^11.6.0" + helmet "^5.0.2" + isomorphic-git "^1.8.0" + jose "^4.6.0" + keyv "^4.0.3" + keyv-memcache "^1.2.5" + knex "^1.0.2" + lodash "^4.17.21" + logform "^2.3.2" + luxon "^2.3.1" + minimatch "^5.0.0" + minimist "^1.2.5" + morgan "^1.10.0" + node-abort-controller "^3.0.1" + node-fetch "^2.6.7" + raw-body "^2.4.1" + selfsigned "^2.0.0" + stoppable "^1.1.0" + tar "^6.1.2" + unzipper "^0.10.11" + winston "^3.2.1" + yn "^4.0.0" + +"@backstage/backend-tasks@^0.3.1": + version "0.3.1" + resolved "https://registry.npmjs.org/@backstage/backend-tasks/-/backend-tasks-0.3.1.tgz#551ff2f5eb41ea1af21c166f957a24512f87909b" + integrity sha512-aCUJl2E0lrmIyzSLKP8kKSZVBA8Wb6cOiHVmoYVrOe9VOvPDx4DZtnvL2iDF+bKfuQlWsvKoAQOcBlgJfyeLeg== + dependencies: + "@backstage/backend-common" "^0.13.3" + "@backstage/config" "^1.0.1" + "@backstage/errors" "^1.0.0" + "@backstage/types" "^1.0.0" + "@types/luxon" "^2.0.4" + cron "^1.8.2" + knex "^1.0.2" + lodash "^4.17.21" + luxon "^2.0.2" + node-abort-controller "^3.0.1" + uuid "^8.0.0" + winston "^3.2.1" + zod "^3.9.5" + "@backstage/catalog-client@^1.0.2": version "1.0.2" resolved "https://registry.npmjs.org/@backstage/catalog-client/-/catalog-client-1.0.2.tgz#8450bb09e6e65114053538d1ab855bc8bee09531" @@ -1487,6 +1559,27 @@ lodash "^4.17.21" uuid "^8.0.0" +"@backstage/config-loader@^1.1.1": + version "1.1.1" + resolved "https://registry.npmjs.org/@backstage/config-loader/-/config-loader-1.1.1.tgz#552bbf331acdee19247f158e62b20649cf07f427" + integrity sha512-LYmX+BPMn74Pyi/tDiELNNDCyKkJIsQL/PKbNw15CEd6LDJI5jOXY9pMxBPlbpg1fYyA46AeF0Yu6V5s/GXa7Q== + dependencies: + "@backstage/cli-common" "^0.1.9" + "@backstage/config" "^1.0.1" + "@backstage/errors" "^1.0.0" + "@backstage/types" "^1.0.0" + "@types/json-schema" "^7.0.6" + ajv "^8.10.0" + chokidar "^3.5.2" + fs-extra "10.1.0" + json-schema "^0.4.0" + json-schema-merge-allof "^0.8.1" + json-schema-traverse "^1.0.0" + node-fetch "^2.6.7" + typescript-json-schema "^0.53.0" + yaml "^1.9.2" + yup "^0.32.9" + "@backstage/core-components@^0.9.0", "@backstage/core-components@^0.9.4": version "0.9.4" resolved "https://registry.npmjs.org/@backstage/core-components/-/core-components-0.9.4.tgz#47e9a305f768a951e0cb0ffa9c1e3c141d06b223" @@ -10283,6 +10376,13 @@ crelt@^1.0.5: resolved "https://registry.npmjs.org/crelt/-/crelt-1.0.5.tgz#57c0d52af8c859e354bace1883eb2e1eb182bb94" integrity sha512-+BO9wPPi+DWTDcNYhr/W90myha8ptzftZT+LwcmUbbok0rcP/fequmFYCw8NMoH7pkAZQzU78b3kYrlua5a9eA== +cron@^1.8.2: + version "1.8.3" + resolved "https://registry.npmjs.org/cron/-/cron-1.8.3.tgz#2a61d7b15848716885834ec56ac072f4b9744ebd" + integrity sha512-JYR/QZFklJCIPndBLfd/2nU1nSlCMrUdtQ2mGLXSVM/qqqEK7DOrFR0gsEiyeqs0PdWrs0ve1ggH4V7XksDwXg== + dependencies: + luxon "^1.23.x" + cron@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/cron/-/cron-2.0.0.tgz#15c6bf37c1cebf6da1d7a688b9ba1c68338bfe6b" From 58927c37841f3024b1c66ec675018aff0f1a0527 Mon Sep 17 00:00:00 2001 From: ivgo Date: Tue, 7 Jun 2022 09:18:40 +0200 Subject: [PATCH 3/6] Fix local dependency Signed-off-by: ivgo --- plugins/catalog-backend-module-gitlab/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/package.json b/plugins/catalog-backend-module-gitlab/package.json index 74615e83fe..048ce85cb8 100644 --- a/plugins/catalog-backend-module-gitlab/package.json +++ b/plugins/catalog-backend-module-gitlab/package.json @@ -35,7 +35,7 @@ "dependencies": { "@backstage/backend-common": "^0.14.0-next.2", "@backstage/catalog-model": "^1.0.3-next.0", - "@backstage/backend-tasks": "^0.3.1", + "@backstage/backend-tasks": "^0.3.2-next.1", "@backstage/config": "^1.0.1", "@backstage/errors": "^1.0.0", "@backstage/integration": "^1.2.1-next.2", From f96cdcb9f0b1f993ef72e8f46a7d3913589bb470 Mon Sep 17 00:00:00 2001 From: ivgo Date: Tue, 7 Jun 2022 11:49:09 +0200 Subject: [PATCH 4/6] Fix typo and add more explanation in the changeset Signed-off-by: ivgo --- .changeset/lazy-zoos-move.md | 62 +++++++++++++++++++++++++++ docs/integrations/gitlab/discovery.md | 4 +- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/.changeset/lazy-zoos-move.md b/.changeset/lazy-zoos-move.md index 1e94ec798f..4bd78cee1d 100644 --- a/.changeset/lazy-zoos-move.md +++ b/.changeset/lazy-zoos-move.md @@ -3,3 +3,65 @@ --- Add a new provider `GitlabDiscoveryEntityProvider` as replacement for `GitlabDiscoveryProcessor` + +In order to migrate from the `GitlabDiscoveryProcessor` you need to apply +the following changes: + +**Before:** + +```yaml +# app-config.yaml + +catalog: + locations: + - type: gitlab-discovery + target: https://company.gitlab.com/prefix/*/catalog-info.yaml +``` + +```ts +/* packages/backend/src/plugins/catalog.ts */ + +import { GitlabDiscoveryProcessor } from '@backstage/plugin-catalog-backend-module-gitlab'; + +const builder = await CatalogBuilder.create(env); +/** ... other processors ... */ +builder.addProcessor( + GitLabDiscoveryProcessor.fromConfig(env.config, { logger: env.logger }), +); +``` + +**After:** + +```yaml +# app-config.yaml + +catalog: + providers: + gitlab: + yourProviderId: # identifies your dataset / provider independent of config changes + host: gitlab-host # Identifies one of the hosts set up in the integrations + branch: main # Optional. Uses `master` as default + group: example-group # Group and subgroup (if needed) to look for repositories + entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` + rules: # Optional. Uses the default rules if not present + - repository: example-repo + allow: [Component, System, Location, Template] +``` + +```ts +/* packages/backend/src/plugins/catalog.ts */ + +import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; + +const builder = await CatalogBuilder.create(env); +/** ... other processors and/or providers ... */ +builder.addEntityProvider( + ...GitlabDiscoveryEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + }), +); +``` diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 986c2aec15..6383e70e64 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -24,7 +24,7 @@ catalog: branch: main # Optional. Uses `master` as default group: example-group # Group and subgroup (if needed) to look for repositories entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` - rules: + rules: # Optional. Uses the default rules if not present - repository: example-repo allow: [Component, System, Location, Template] ``` @@ -42,7 +42,7 @@ Once you've done that, you'll also need to add the segment below to `packages/ba ```ts /* packages/backend/src/plugins/catalog.ts */ -import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-aws'; +import { GitlabDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab'; const builder = await CatalogBuilder.create(env); /** ... other processors and/or providers ... */ From 90450c925b18a14705abf8545b60e3c3776c83a8 Mon Sep 17 00:00:00 2001 From: ivgo Date: Tue, 7 Jun 2022 17:34:23 +0200 Subject: [PATCH 5/6] Fix some typos & update yarn.lock after release Signed-off-by: ivgo --- .../catalog-backend-module-gitlab/config.d.ts | 2 - .../src/lib/client.ts | 9 +- .../GitlabDiscoveryEntityProvider.ts | 2 +- yarn.lock | 100 ------------------ 4 files changed, 5 insertions(+), 108 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/config.d.ts b/plugins/catalog-backend-module-gitlab/config.d.ts index 11c6830e63..5b0d42c81d 100644 --- a/plugins/catalog-backend-module-gitlab/config.d.ts +++ b/plugins/catalog-backend-module-gitlab/config.d.ts @@ -22,8 +22,6 @@ export interface Config { providers?: { /** * GitlabDiscoveryEntityProvider configuration - * - * Uses "default" as default id for the single config variant. */ gitlab?: Record< string, diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 25384f3dd6..adf0a78e8f 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -64,12 +64,11 @@ export class GitLabClient { } /** - * Checks if the catalog file is present in the repository or not. + * General existence check. * - * @param projectPath The path to the project - * @param branch The branch used to injest entities to the catalog - * @param filePath The path to the catalog file - * @returns `true` if the file exists, `false` otherwise + * @param projectPath - The path to the project + * @param branch - The branch used to search + * @param filePath - The path to the file */ async hasFile( projectPath: string, diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index 7a30aa83da..1ecf274c3b 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -39,7 +39,7 @@ type Result = { }; /** - * Extracts repositories out of an GitLab instance. + * Discovers entity definition files in the groups of a Gitlab instance. * @public */ export class GitlabDiscoveryEntityProvider implements EntityProvider { diff --git a/yarn.lock b/yarn.lock index fb4b8f867c..c7795b8004 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1465,78 +1465,6 @@ "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" -"@backstage/backend-common@^0.13.3": - version "0.13.5" - resolved "https://registry.npmjs.org/@backstage/backend-common/-/backend-common-0.13.5.tgz#9ff55e1da2a9cf65d0c4782cb360f9b6abd0754c" - integrity sha512-37kLw1BolfstantREuawbyRKXD+xKHUta3A63j0868JZPxRSCfiysBzF3uAorH/3WIh0p2lswutmunbVQFsXDA== - dependencies: - "@backstage/cli-common" "^0.1.9" - "@backstage/config" "^1.0.1" - "@backstage/config-loader" "^1.1.1" - "@backstage/errors" "^1.0.0" - "@backstage/integration" "^1.2.0" - "@backstage/types" "^1.0.0" - "@google-cloud/storage" "^5.8.0" - "@keyv/redis" "^2.2.3" - "@manypkg/get-packages" "^1.1.3" - "@octokit/rest" "^18.5.3" - "@types/cors" "^2.8.6" - "@types/dockerode" "^3.3.0" - "@types/express" "^4.17.6" - "@types/luxon" "^2.0.4" - "@types/webpack-env" "^1.15.2" - archiver "^5.0.2" - aws-sdk "^2.840.0" - base64-stream "^1.0.0" - compression "^1.7.4" - concat-stream "^2.0.0" - cors "^2.8.5" - dockerode "^3.3.1" - express "^4.17.1" - express-promise-router "^4.1.0" - fs-extra "10.1.0" - git-url-parse "^11.6.0" - helmet "^5.0.2" - isomorphic-git "^1.8.0" - jose "^4.6.0" - keyv "^4.0.3" - keyv-memcache "^1.2.5" - knex "^1.0.2" - lodash "^4.17.21" - logform "^2.3.2" - luxon "^2.3.1" - minimatch "^5.0.0" - minimist "^1.2.5" - morgan "^1.10.0" - node-abort-controller "^3.0.1" - node-fetch "^2.6.7" - raw-body "^2.4.1" - selfsigned "^2.0.0" - stoppable "^1.1.0" - tar "^6.1.2" - unzipper "^0.10.11" - winston "^3.2.1" - yn "^4.0.0" - -"@backstage/backend-tasks@^0.3.1": - version "0.3.1" - resolved "https://registry.npmjs.org/@backstage/backend-tasks/-/backend-tasks-0.3.1.tgz#551ff2f5eb41ea1af21c166f957a24512f87909b" - integrity sha512-aCUJl2E0lrmIyzSLKP8kKSZVBA8Wb6cOiHVmoYVrOe9VOvPDx4DZtnvL2iDF+bKfuQlWsvKoAQOcBlgJfyeLeg== - dependencies: - "@backstage/backend-common" "^0.13.3" - "@backstage/config" "^1.0.1" - "@backstage/errors" "^1.0.0" - "@backstage/types" "^1.0.0" - "@types/luxon" "^2.0.4" - cron "^1.8.2" - knex "^1.0.2" - lodash "^4.17.21" - luxon "^2.0.2" - node-abort-controller "^3.0.1" - uuid "^8.0.0" - winston "^3.2.1" - zod "^3.9.5" - "@backstage/catalog-client@^1.0.2": version "1.0.2" resolved "https://registry.npmjs.org/@backstage/catalog-client/-/catalog-client-1.0.2.tgz#8450bb09e6e65114053538d1ab855bc8bee09531" @@ -1559,27 +1487,6 @@ lodash "^4.17.21" uuid "^8.0.0" -"@backstage/config-loader@^1.1.1": - version "1.1.1" - resolved "https://registry.npmjs.org/@backstage/config-loader/-/config-loader-1.1.1.tgz#552bbf331acdee19247f158e62b20649cf07f427" - integrity sha512-LYmX+BPMn74Pyi/tDiELNNDCyKkJIsQL/PKbNw15CEd6LDJI5jOXY9pMxBPlbpg1fYyA46AeF0Yu6V5s/GXa7Q== - dependencies: - "@backstage/cli-common" "^0.1.9" - "@backstage/config" "^1.0.1" - "@backstage/errors" "^1.0.0" - "@backstage/types" "^1.0.0" - "@types/json-schema" "^7.0.6" - ajv "^8.10.0" - chokidar "^3.5.2" - fs-extra "10.1.0" - json-schema "^0.4.0" - json-schema-merge-allof "^0.8.1" - json-schema-traverse "^1.0.0" - node-fetch "^2.6.7" - typescript-json-schema "^0.53.0" - yaml "^1.9.2" - yup "^0.32.9" - "@backstage/core-components@^0.9.0", "@backstage/core-components@^0.9.4": version "0.9.4" resolved "https://registry.npmjs.org/@backstage/core-components/-/core-components-0.9.4.tgz#47e9a305f768a951e0cb0ffa9c1e3c141d06b223" @@ -10376,13 +10283,6 @@ crelt@^1.0.5: resolved "https://registry.npmjs.org/crelt/-/crelt-1.0.5.tgz#57c0d52af8c859e354bace1883eb2e1eb182bb94" integrity sha512-+BO9wPPi+DWTDcNYhr/W90myha8ptzftZT+LwcmUbbok0rcP/fequmFYCw8NMoH7pkAZQzU78b3kYrlua5a9eA== -cron@^1.8.2: - version "1.8.3" - resolved "https://registry.npmjs.org/cron/-/cron-1.8.3.tgz#2a61d7b15848716885834ec56ac072f4b9744ebd" - integrity sha512-JYR/QZFklJCIPndBLfd/2nU1nSlCMrUdtQ2mGLXSVM/qqqEK7DOrFR0gsEiyeqs0PdWrs0ve1ggH4V7XksDwXg== - dependencies: - luxon "^1.23.x" - cron@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/cron/-/cron-2.0.0.tgz#15c6bf37c1cebf6da1d7a688b9ba1c68338bfe6b" From 6b1ed2292d4d01fcb13a99faccc522de1be370bc Mon Sep 17 00:00:00 2001 From: ivgo Date: Thu, 9 Jun 2022 14:50:01 +0200 Subject: [PATCH 6/6] Remove rules & update some tsc errors Signed-off-by: ivgo --- .changeset/lazy-zoos-move.md | 3 -- .changeset/perfect-mangos-allow.md | 5 --- docs/integrations/gitlab/discovery.md | 3 -- .../src/lib/types.ts | 2 +- .../GitlabDiscoveryEntityProvider.ts | 2 +- .../src/ingestion/CatalogRules.ts | 45 ------------------- 6 files changed, 2 insertions(+), 58 deletions(-) delete mode 100644 .changeset/perfect-mangos-allow.md diff --git a/.changeset/lazy-zoos-move.md b/.changeset/lazy-zoos-move.md index 4bd78cee1d..69bd517438 100644 --- a/.changeset/lazy-zoos-move.md +++ b/.changeset/lazy-zoos-move.md @@ -43,9 +43,6 @@ catalog: branch: main # Optional. Uses `master` as default group: example-group # Group and subgroup (if needed) to look for repositories entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` - rules: # Optional. Uses the default rules if not present - - repository: example-repo - allow: [Component, System, Location, Template] ``` ```ts diff --git a/.changeset/perfect-mangos-allow.md b/.changeset/perfect-mangos-allow.md deleted file mode 100644 index 0fe75e009c..0000000000 --- a/.changeset/perfect-mangos-allow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-backend': patch ---- - -Add support to custom rules for `GitlabDiscoveryEntityProvider` diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 6383e70e64..ea151e9415 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -24,9 +24,6 @@ catalog: branch: main # Optional. Uses `master` as default group: example-group # Group and subgroup (if needed) to look for repositories entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml` - rules: # Optional. Uses the default rules if not present - - repository: example-repo - allow: [Component, System, Location, Template] ``` As this provider is not one of the default providers, you will first need to install diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index fa2350487f..69de9d28c7 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -26,7 +26,7 @@ export type GitLabProject = { archived: boolean; last_activity_at: string; web_url: string; - path_with_namespace: string; + path_with_namespace?: string; }; export type GitlabProviderConfig = { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts index 1ecf274c3b..36c3ee90be 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts @@ -160,7 +160,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider { const project_branch = project.default_branch ?? this.config.branch; const projectHasFile: boolean = await client.hasFile( - project.path_with_namespace, + project.path_with_namespace ?? '', project_branch, this.config.catalogFile, ); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 72eae22b11..587a0bd7d5 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -18,7 +18,6 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import path from 'path'; import { LocationSpec } from '../api'; -import { ScmIntegrations } from '@backstage/integration'; /** * Rules to apply to catalog entities. @@ -120,18 +119,6 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { rules.push(...locationRules); } - if (config.has('catalog.providers')) { - const providersConf = config.getConfig('catalog.providers'); - const providerList = providersConf.keys(); - providerList.forEach(provider => { - if (provider === 'gitlab') { - rules.push( - ...getGitlabRules(config, providersConf.getConfig(provider)), - ); - } - }); - } - return new DefaultCatalogRulesEnforcer(rules); } @@ -200,35 +187,3 @@ function resolveTarget(type: string, target: string): string { return path.resolve(target); } - -function getGitlabRules(initialConfig: Config, config: Config): CatalogRule[] { - return config.keys().flatMap(id => { - const gitlabConf = config.getConfig(id); - if (!gitlabConf.has('rules')) { - return []; - } - - const integrations = ScmIntegrations.fromConfig(initialConfig).gitlab; - const gitlabHost = gitlabConf.getString('host'); - const integration = integrations.byHost(gitlabHost); - if (!integration) { - throw new Error( - `No gitlab integration found that matches host ${gitlabHost}`, - ); - } - - const type = `url`; - const branch = gitlabConf.getOptionalString('branch') ?? 'master'; - const entityFilename = - gitlabConf.getOptionalString('entityFilename') ?? 'catalog-info.yaml'; - - return gitlabConf.getConfigArray('rules').map(ruleConf => { - const repoName = ruleConf.getString('repository'); - const target = `${integration.config.baseUrl}/${id}/${repoName}/-/blob/${branch}/${entityFilename}`; - return { - allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), - locations: [{ type, target }], - }; - }); - }); -}