From 1136ac88b604946f522a6e08333f5e271580384b Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 15:03:51 +0200 Subject: [PATCH 1/9] First part of GitLabDiscoveryProcessor (no target parsing yet) Signed-off-by: Roy Jacobs --- .../GitLabDiscoveryProcessor.test.ts | 252 ++++++++++++++++++ .../processors/GitLabDiscoveryProcessor.ts | 129 +++++++++ .../src/ingestion/processors/gitlab/client.ts | 94 +++++++ .../src/ingestion/processors/gitlab/index.ts | 18 ++ .../src/ingestion/processors/gitlab/types.ts | 23 ++ .../src/ingestion/processors/index.ts | 1 + .../src/next/NextCatalogBuilder.ts | 2 + .../src/service/CatalogBuilder.ts | 2 + 8 files changed, 521 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..5eb5c51869 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -0,0 +1,252 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { + GitLabDiscoveryProcessor, +} from './GitLabDiscoveryProcessor'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; +import { GitLabProject } from "./gitlab"; + +const server = setupServer(); + +function setupFakeGitLab( + callback: (request: { + page: number; + }) => { data: GitLabProject[]; nextPage?: number }, +) { + server.use( + rest.get('https://gitlab.fake/api/v4/projects', (req, res, ctx) => { + if (req.headers.get('private-token') !== 'test-token') { + return res(ctx.status(401), ctx.json({})); + } + const page = req.url.searchParams.get('page'); + const response = callback({ + page: parseInt(page!, 10), + }); + + // Filter the fake results based on the `last_activity_after` parameter + const last_activity_after = req.url.searchParams.get( + 'last_activity_after', + ); + const filteredData = response.data.filter( + v => + !last_activity_after || + Date.parse(v.last_activity_at) >= Date.parse(last_activity_after), + ); + + return res( + ctx.set('x-next-page', response.nextPage?.toString() ?? ''), + ctx.json(filteredData), + ); + }), + ); +} + +function getConfig(): any { + return { + backend: { + cache: { store: 'memory' }, + }, + integrations: { + gitlab: [ + { + host: 'gitlab.fake', + apiBaseUrl: 'https://gitlab.fake/api/v4', + token: 'test-token', + }, + ], + }, + }; +} + +function getProcessor(config?: any): GitLabDiscoveryProcessor { + return GitLabDiscoveryProcessor.fromConfig( + new ConfigReader(config || getConfig()), + { + logger: getVoidLogger(), + }, + ); +} + +describe('GitlabDiscoveryProcessor', () => { + beforeAll(() => { + server.listen(); + jest.useFakeTimers('modern'); + jest.setSystemTime(new Date('2001-01-01T12:34:56Z')); + }); + afterEach(() => server.resetHandlers()); + afterAll(() => { + server.close(); + jest.useRealTimers(); + }); + + const location: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/north-star', + }; + + describe('handles repositories', () => { + it('pages through all repositories', async () => { + const processor = getProcessor(); + setupFakeGitLab(request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + nextPage: 2, + }; + case 2: + return { + data: [ + { + id: 2, + archived: false, + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/2', + }, + { + id: 3, + archived: true, // ARCHIVED + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/3', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(location, false, e => { + result.push(e); + }); + expect(result).toEqual([ + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/1/-/blob/main/catalog-info.yaml', + }, + optional: true, + }, + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml', + }, + optional: true, + }, + ]); + }); + + it('uses the previous scan timestamp to filter', async () => { + const processor = getProcessor(); + setupFakeGitLab(request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2000-01-01T00:00:00Z', + web_url: 'https://gitlab.fake/1', + }, + { + id: 2, + archived: false, + default_branch: 'main', + last_activity_at: '2002-01-01T00:00:00Z', + web_url: 'https://gitlab.fake/2', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + + // First scan should find all repos, since no last activity was cached + await processor.readLocation(location, false, e => { + result.push(e); + }); + expect(result).toHaveLength(2); + + // Second scan should have used the mocked Date to set the last scanned time to 2001 + // This should result in only the second repo being scanned, since that has a timestamp of 2002 + const result2: any[] = []; + await processor.readLocation(location, false, e => { + result2.push(e); + }); + expect(result2).toHaveLength(1); + }); + }); + + describe('handles failure', () => { + it('invalid token', async () => { + // Setup an empty fake gitlab, since we don't care about actual results + setupFakeGitLab(_ => { + return { + data: [], + }; + }); + + const config = getConfig(); + config.integrations.gitlab[0].token = 'invalid'; + await expect( + getProcessor(config).readLocation(location, false, _ => {}), + ).rejects.toThrow(/Unauthorized/); + }); + + it('missing integration', async () => { + const config = getConfig(); + delete config.integrations; + await expect( + getProcessor(config).readLocation(location, false, _ => {}), + ).rejects.toThrow(/no GitLab integration/); + }); + + it('location type', async () => { + const incorrectLocation: LocationSpec = { + type: 'something-that-is-not-gitlab-discovery', + target: 'https://gitlab.fake/north-star', + }; + + await expect( + getProcessor().readLocation(incorrectLocation, false, _ => {}), + ).resolves.toBeFalsy(); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts new file mode 100644 index 0000000000..b3cce6e711 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + ScmIntegrations, +} from '@backstage/integration'; +import { Logger } from 'winston'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { GitLabClient, GitLabProject, paginated } from "./gitlab"; +import { CacheClient, CacheManager, PluginCacheManager } from "@backstage/backend-common"; + +/** + * Extracts repositories out of a GitLab org. + */ +export class GitLabDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + private readonly cache: CacheClient; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + const pluginCache = CacheManager.fromConfig(config).forPlugin( + 'gitlab-discovery', + ); + + return new GitLabDiscoveryProcessor({ + ...options, + integrations, + pluginCache, + }); + } + + private constructor(options: { + integrations: ScmIntegrations; + pluginCache: PluginCacheManager; + logger: Logger; + }) { + this.integrations = options.integrations; + this.cache = options.pluginCache.getClient(); + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'gitlab-discovery') { + return false; + } + + const integration = this.integrations.gitlab.byUrl(location.target); + if (!integration) { + throw new Error( + `There is no GitLab integration that matches ${location.target}. Please add a configuration entry for it under integrations.gitlab`, + ); + } + + const client = new GitLabClient({ + config: integration.config, + logger: this.logger, + }); + const startTimestamp = Date.now(); + this.logger.info(`Reading GitLab projects from ${location.target}`); + + const projects = paginated(options => client.listProjects(options), { + last_activity_after: await this.updateLastActivity(), + page: 1, + }); + + const result: Result = { + scanned: 0, + matches: [], + }; + for await (const project of projects) { + result.scanned++; + if (!project.archived) { + result.matches.push(project); + } + } + + for (const project of result.matches) { + emit( + results.location( + { + type: 'url', + // The format expected by the GitLabUrlReader: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + target: `${project.web_url}/-/blob/${project.default_branch}/catalog-info.yaml`, + }, + true, + ), + ); + } + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${result.scanned} GitLab repositories in ${duration} seconds`, + ); + + return true; + } + + async updateLastActivity(): Promise { + const lastActivity = await this.cache.get('last-activity'); + await this.cache.set('last-activity', new Date().toISOString()); + return lastActivity as string | undefined; + } +} + +type Result = { + scanned: number; + matches: GitLabProject[]; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts new file mode 100644 index 0000000000..fb7763dd1c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'cross-fetch'; +import { + getGitLabRequestOptions, + GitLabIntegrationConfig, +} from '@backstage/integration'; +import { Logger } from 'winston'; + +export class GitLabClient { + private readonly config: GitLabIntegrationConfig; + private readonly logger: Logger; + + constructor(options: { config: GitLabIntegrationConfig; logger: Logger }) { + this.config = options.config; + this.logger = options.logger; + } + + async listProjects(options?: ListOptions): Promise> { + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); + } + + private async pagedRequest( + endpoint: string, + options?: ListOptions, + ): Promise> { + const request = new URL(endpoint); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } + } + + this.logger.debug(`Fetching: ${request.toString()}`); + const response = await fetch( + request.toString(), + getGitLabRequestOptions(this.config), + ); + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return response.json().then(items => { + const nextPage = response.headers.get('x-next-page'); + + return { + items, + nextPage: nextPage ? Number(nextPage) : null, + } as PagedResponse; + }); + } +} + +export type ListOptions = { + [key: string]: string | number | undefined; + per_page?: number | undefined; + page?: number | undefined; +}; + +export type PagedResponse = { + items: T[]; + nextPage?: number; +}; + +export async function* paginated( + request: (options: ListOptions) => Promise>, + options: ListOptions, +) { + let res; + do { + res = await request(options); + options.page = res.nextPage; + for (const item of res.items) { + yield item; + } + } while (res.nextPage); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts new file mode 100644 index 0000000000..a6c6419d21 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitLabClient, paginated } from './client'; +export type { GitLabProject } from "./types"; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts new file mode 100644 index 0000000000..a66411ddc8 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type GitLabProject = { + id: number; + default_branch: string; + archived: boolean; + last_activity_at: string; + web_url: string; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 6a44ea0a8f..fe4d374829 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -26,6 +26,7 @@ export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; +export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; export type { PlaceholderResolver } from './PlaceholderProcessor'; diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index fb12077508..88fcc27055 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -50,6 +50,7 @@ import { FileReaderProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, + GitLabDiscoveryProcessor, PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, @@ -372,6 +373,7 @@ export class NextCatalogBuilder { BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), + GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new AnnotateLocationEntityProcessor({ integrations }), diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 76b4af7450..70c1890fbf 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -46,6 +46,7 @@ import { FileReaderProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, + GitLabDiscoveryProcessor, HigherOrderOperation, HigherOrderOperations, LocationEntityProcessor, @@ -316,6 +317,7 @@ export class CatalogBuilder { BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), + GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new LocationEntityProcessor({ integrations }), From a487f61564c3ccdc6938f8990fb1313b367dd827 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:24:07 +0200 Subject: [PATCH 2/9] Parse URLs and allow filtering based on group/subgroup Signed-off-by: Roy Jacobs --- .../GitLabDiscoveryProcessor.test.ts | 160 +++++++++++++++--- .../processors/GitLabDiscoveryProcessor.ts | 71 ++++++-- .../src/ingestion/processors/gitlab/client.ts | 15 +- 3 files changed, 210 insertions(+), 36 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts index 5eb5c51869..a0e44d7648 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -17,28 +17,47 @@ import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; -import { - GitLabDiscoveryProcessor, -} from './GitLabDiscoveryProcessor'; +import { GitLabDiscoveryProcessor, parseUrl } from './GitLabDiscoveryProcessor'; import { setupServer } from 'msw/node'; import { rest } from 'msw'; -import { GitLabProject } from "./gitlab"; +import { GitLabProject } from './gitlab'; const server = setupServer(); -function setupFakeGitLab( - callback: (request: { - page: number; - }) => { data: GitLabProject[]; nextPage?: number }, +const PROJECTS_URL = 'https://gitlab.fake/api/v4/projects'; +const GROUP_PROJECTS_URL = + 'https://gitlab.fake/api/v4/groups/group%2Fsubgroup/projects'; + +const PROJECT_LOCATION: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/blob/*/catalog-info.yaml', +}; +const PROJECT_LOCATION_MASTER_BRANCH: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/blob/master/catalog-info.yaml', +}; +const GROUP_LOCATION: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/group/subgroup/blob/*/catalog-info.yaml', +}; + +function setupFakeServer( + url: string, + callback: (request: { page: number; include_subgroups: boolean }) => { + data: GitLabProject[]; + nextPage?: number; + }, ) { server.use( - rest.get('https://gitlab.fake/api/v4/projects', (req, res, ctx) => { + rest.get(url, (req, res, ctx) => { if (req.headers.get('private-token') !== 'test-token') { return res(ctx.status(401), ctx.json({})); } const page = req.url.searchParams.get('page'); + const include_subgroups = req.url.searchParams.get('include_subgroups'); const response = callback({ page: parseInt(page!, 10), + include_subgroups: include_subgroups === 'true', }); // Filter the fake results based on the `last_activity_after` parameter @@ -97,15 +116,43 @@ describe('GitlabDiscoveryProcessor', () => { jest.useRealTimers(); }); - const location: LocationSpec = { - type: 'gitlab-discovery', - target: 'https://gitlab.fake/north-star', - }; + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect( + parseUrl('https://gitlab.com/group/subgroup/blob/master/catalog.yaml'), + ).toEqual({ + group: 'group/subgroup', + host: 'gitlab.com', + branch: 'master', + catalogPath: 'catalog.yaml', + }); + expect( + parseUrl('https://gitlab.com/blob/*/subfolder/catalog.yaml'), + ).toEqual({ + group: undefined, + host: 'gitlab.com', + branch: '*', + catalogPath: 'subfolder/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://gitlab.com')).toThrow(); + expect(() => parseUrl('https://gitlab.com//')).toThrow(); + expect(() => parseUrl('https://gitlab.com/foo')).toThrow(); + expect(() => parseUrl('https://gitlab.com//foo')).toThrow(); + expect(() => parseUrl('https://gitlab.com/org/teams')).toThrow(); + expect(() => parseUrl('https://gitlab.com/org//teams')).toThrow(); + expect(() => + parseUrl('https://gitlab.com/org//teams/blob/catalog.yaml'), + ).toThrow(); + }); + }); describe('handles repositories', () => { it('pages through all repositories', async () => { const processor = getProcessor(); - setupFakeGitLab(request => { + setupFakeServer(PROJECTS_URL, request => { switch (request.page) { case 1: return { @@ -145,7 +192,7 @@ describe('GitlabDiscoveryProcessor', () => { }); const result: any[] = []; - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result.push(e); }); expect(result).toEqual([ @@ -168,9 +215,78 @@ describe('GitlabDiscoveryProcessor', () => { ]); }); + it('can force a branch name', async () => { + const processor = getProcessor(); + setupFakeServer(PROJECTS_URL, request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(PROJECT_LOCATION_MASTER_BRANCH, false, e => { + result.push(e); + }); + expect(result).toEqual([ + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/1/-/blob/master/catalog-info.yaml', + }, + optional: true, + }, + ]); + }); + + it('can filter based on group', async () => { + const processor = getProcessor(); + setupFakeServer(GROUP_PROJECTS_URL, request => { + if (!request.include_subgroups) { + throw new Error('include_subgroups should be set'); + } + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(GROUP_LOCATION, false, e => { + result.push(e); + }); + // If everything was set up correctly, we should have received the fake repo specified above + expect(result).toHaveLength(1); + }); + it('uses the previous scan timestamp to filter', async () => { const processor = getProcessor(); - setupFakeGitLab(request => { + setupFakeServer(PROJECTS_URL, request => { switch (request.page) { case 1: return { @@ -199,7 +315,7 @@ describe('GitlabDiscoveryProcessor', () => { const result: any[] = []; // First scan should find all repos, since no last activity was cached - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result.push(e); }); expect(result).toHaveLength(2); @@ -207,7 +323,7 @@ describe('GitlabDiscoveryProcessor', () => { // Second scan should have used the mocked Date to set the last scanned time to 2001 // This should result in only the second repo being scanned, since that has a timestamp of 2002 const result2: any[] = []; - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result2.push(e); }); expect(result2).toHaveLength(1); @@ -217,7 +333,7 @@ describe('GitlabDiscoveryProcessor', () => { describe('handles failure', () => { it('invalid token', async () => { // Setup an empty fake gitlab, since we don't care about actual results - setupFakeGitLab(_ => { + setupFakeServer(PROJECTS_URL, _ => { return { data: [], }; @@ -226,7 +342,7 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); config.integrations.gitlab[0].token = 'invalid'; await expect( - getProcessor(config).readLocation(location, false, _ => {}), + getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/Unauthorized/); }); @@ -234,14 +350,14 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); delete config.integrations; await expect( - getProcessor(config).readLocation(location, false, _ => {}), + getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/no GitLab integration/); }); it('location type', async () => { const incorrectLocation: LocationSpec = { type: 'something-that-is-not-gitlab-discovery', - target: 'https://gitlab.fake/north-star', + target: 'https://gitlab.fake/oh-dear', }; await expect( diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index b3cce6e711..5603a1aa7f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -16,17 +16,19 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { - ScmIntegrations, -} from '@backstage/integration'; +import { ScmIntegrations } from '@backstage/integration'; import { Logger } from 'winston'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { GitLabClient, GitLabProject, paginated } from "./gitlab"; -import { CacheClient, CacheManager, PluginCacheManager } from "@backstage/backend-common"; +import { GitLabClient, GitLabProject, paginated } from './gitlab'; +import { + CacheClient, + CacheManager, + PluginCacheManager, +} from '@backstage/backend-common'; /** - * Extracts repositories out of a GitLab org. + * Extracts repositories out of an GitLab instance. */ export class GitLabDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; @@ -35,9 +37,8 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { static fromConfig(config: Config, options: { logger: Logger }) { const integrations = ScmIntegrations.fromConfig(config); - const pluginCache = CacheManager.fromConfig(config).forPlugin( - 'gitlab-discovery', - ); + const pluginCache = + CacheManager.fromConfig(config).forPlugin('gitlab-discovery'); return new GitLabDiscoveryProcessor({ ...options, @@ -65,10 +66,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { return false; } - const integration = this.integrations.gitlab.byUrl(location.target); + const { group, host, branch, catalogPath } = parseUrl(location.target); + + const integration = this.integrations.gitlab.byUrl(`https://${host}`); if (!integration) { throw new Error( - `There is no GitLab integration that matches ${location.target}. Please add a configuration entry for it under integrations.gitlab`, + `There is no GitLab integration that matches ${host}. Please add a configuration entry for it under integrations.gitlab`, ); } @@ -80,6 +83,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { this.logger.info(`Reading GitLab projects from ${location.target}`); const projects = paginated(options => client.listProjects(options), { + group, last_activity_after: await this.updateLastActivity(), page: 1, }); @@ -96,12 +100,19 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { } for (const project of result.matches) { + const project_branch = branch === '*' ? project.default_branch : branch; + emit( results.location( { type: 'url', - // The format expected by the GitLabUrlReader: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath - target: `${project.web_url}/-/blob/${project.default_branch}/catalog-info.yaml`, + // The format expected by the GitLabUrlReader: + // https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + // + // This unfortunately will trigger another API call in `getGitLabFileFetchUrl` to get the project ID. + // The alternative is using the `buildRawUrl` function, which does not support subgroups, so providing a raw + // URL here won't work either. + target: `${project.web_url}/-/blob/${project_branch}/${catalogPath}`, }, true, ), @@ -127,3 +138,37 @@ type Result = { scanned: number; matches: GitLabProject[]; }; + +/* + * Helpers + */ + +export function parseUrl(urlString: string): { + group?: string; + host: string; + branch: string; + catalogPath: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // (/group/subgroup)/blob/branch|*/filepath + const blobIndex = path.findIndex(p => p === 'blob'); + if (blobIndex !== -1 && path.length > blobIndex + 2) { + const group = + blobIndex > 0 ? path.slice(0, blobIndex).join('/') : undefined; + + return { + group, + host: url.host, + branch: decodeURIComponent(path[blobIndex + 1]), + catalogPath: decodeURIComponent(path.slice(blobIndex + 2).join('/')), + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +export function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts index fb7763dd1c..8781e071f7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -31,6 +31,18 @@ export class GitLabClient { } async listProjects(options?: ListOptions): Promise> { + if (options?.group) { + return this.pagedRequest( + `${this.config.apiBaseUrl}/groups/${encodeURIComponent( + options?.group, + )}/projects`, + { + ...options, + include_subgroups: true, + }, + ); + } + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); } @@ -69,7 +81,8 @@ export class GitLabClient { } export type ListOptions = { - [key: string]: string | number | undefined; + [key: string]: string | number | boolean | undefined; + group?: string; per_page?: number | undefined; page?: number | undefined; }; From 29f6c7541a4f10dd3844924c7ed97a586be05359 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:28:36 +0200 Subject: [PATCH 3/9] Add documentation for GitLab discovery Signed-off-by: Roy Jacobs --- docs/integrations/gitlab/discovery.md | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/integrations/gitlab/discovery.md diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md new file mode 100644 index 0000000000..ade575b74d --- /dev/null +++ b/docs/integrations/gitlab/discovery.md @@ -0,0 +1,36 @@ +--- +id: discovery +title: GitLab Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in GitLab +--- + +The GitLab integration has a special discovery processor for discovering catalog +entities from GitLab. The processor will crawl the GitLab instance and register +entities matching the configured path. This can be useful as an alternative to +static locations or manually adding things to the catalog. + +To use the discovery processor, you'll need a GitLab integration +[set up](locations.md) with a `token`. Then you can add a location target to the +catalog configuration: + +```yaml +catalog: + locations: + - type: gitlab-discovery + target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml +``` + +Note the `gitlab-discovery` type, as this is not a regular `url` processor. + +The target is composed of three parts: + +- The base URL, `https://gitlab.com` in this case +- The group path, `group/subgroup` in this case. This is optional: If you omit + this path the processor will scan the entire GitLab instance instead. +- The path within each repository to find the catalog YAML file. This will + usually be `/blob/main/catalog-info.yaml`, `/blob/master/catalog-info.yaml` or + a similar variation for catalog files stored in the root directory of each + repository. If you want to use the repository's default branch use the `*` + wildcard, e.g.: `/blob/*/catalog-info.yaml` From 96785dce38ba99b317677decb24585a9e5c22bef Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:31:57 +0200 Subject: [PATCH 4/9] Add changeset for GitLab discovery Signed-off-by: Roy Jacobs --- .changeset/lucky-gifts-help.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lucky-gifts-help.md diff --git a/.changeset/lucky-gifts-help.md b/.changeset/lucky-gifts-help.md new file mode 100644 index 0000000000..1bc100d824 --- /dev/null +++ b/.changeset/lucky-gifts-help.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added GitLabDiscoveryProcessor, which allows catalog discovery from a GitLab instance From 017f2f6c7e2a2bbf37f48776da976b53e9cca1e3 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:44:33 +0200 Subject: [PATCH 5/9] Forgot to run 'Prettier' Signed-off-by: Roy Jacobs --- .../catalog-backend/src/ingestion/processors/gitlab/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts index a6c6419d21..1df3d2cb84 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts @@ -15,4 +15,4 @@ */ export { GitLabClient, paginated } from './client'; -export type { GitLabProject } from "./types"; +export type { GitLabProject } from './types'; From a6ae6115d3b23e6688fcc7081d25a7ce8b8a2ae3 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 17:08:21 +0200 Subject: [PATCH 6/9] Update api-report.md Signed-off-by: Roy Jacobs --- plugins/catalog-backend/api-report.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 7286855d4e..0c7b08d94c 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -771,6 +771,27 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "GitLabDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export class GitLabDiscoveryProcessor implements CatalogProcessor { + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger_2; + }, + ): GitLabDiscoveryProcessor; + // (undocumented) + readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise; + // (undocumented) + updateLastActivity(): Promise; +} + // Warning: (ae-missing-release-tag) "HigherOrderOperation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) From 32df8a75f382679cca3c0ea6bccc0947259c40d6 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Fri, 20 Aug 2021 09:39:06 +0200 Subject: [PATCH 7/9] Remove unused function Signed-off-by: Roy Jacobs --- .../src/ingestion/processors/GitLabDiscoveryProcessor.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index 5603a1aa7f..ccd9f17050 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -168,7 +168,3 @@ export function parseUrl(urlString: string): { throw new Error(`Failed to parse ${urlString}`); } - -export function escapeRegExp(str: string): RegExp { - return new RegExp(`^${str.replace(/\*/g, '.*')}$`); -} From 72505a907ff014dd19c9c819ebf7fe5ed094bb1d Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Fri, 20 Aug 2021 09:46:18 +0200 Subject: [PATCH 8/9] Reduce log level to avoid spamming the logs Signed-off-by: Roy Jacobs --- .../src/ingestion/processors/GitLabDiscoveryProcessor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index ccd9f17050..445f02829e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -80,7 +80,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { logger: this.logger, }); const startTimestamp = Date.now(); - this.logger.info(`Reading GitLab projects from ${location.target}`); + this.logger.debug(`Reading GitLab projects from ${location.target}`); const projects = paginated(options => client.listProjects(options), { group, From 04ef773b94943a9828f270a12ae5253263964b76 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 24 Aug 2021 09:49:43 +0200 Subject: [PATCH 9/9] Reduce changset level to 'patch' Signed-off-by: Roy Jacobs --- .changeset/lucky-gifts-help.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/lucky-gifts-help.md b/.changeset/lucky-gifts-help.md index 1bc100d824..3ed6b12498 100644 --- a/.changeset/lucky-gifts-help.md +++ b/.changeset/lucky-gifts-help.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-backend': patch --- Added GitLabDiscoveryProcessor, which allows catalog discovery from a GitLab instance