From a487f61564c3ccdc6938f8990fb1313b367dd827 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:24:07 +0200 Subject: [PATCH] Parse URLs and allow filtering based on group/subgroup Signed-off-by: Roy Jacobs --- .../GitLabDiscoveryProcessor.test.ts | 160 +++++++++++++++--- .../processors/GitLabDiscoveryProcessor.ts | 71 ++++++-- .../src/ingestion/processors/gitlab/client.ts | 15 +- 3 files changed, 210 insertions(+), 36 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts index 5eb5c51869..a0e44d7648 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -17,28 +17,47 @@ import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; -import { - GitLabDiscoveryProcessor, -} from './GitLabDiscoveryProcessor'; +import { GitLabDiscoveryProcessor, parseUrl } from './GitLabDiscoveryProcessor'; import { setupServer } from 'msw/node'; import { rest } from 'msw'; -import { GitLabProject } from "./gitlab"; +import { GitLabProject } from './gitlab'; const server = setupServer(); -function setupFakeGitLab( - callback: (request: { - page: number; - }) => { data: GitLabProject[]; nextPage?: number }, +const PROJECTS_URL = 'https://gitlab.fake/api/v4/projects'; +const GROUP_PROJECTS_URL = + 'https://gitlab.fake/api/v4/groups/group%2Fsubgroup/projects'; + +const PROJECT_LOCATION: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/blob/*/catalog-info.yaml', +}; +const PROJECT_LOCATION_MASTER_BRANCH: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/blob/master/catalog-info.yaml', +}; +const GROUP_LOCATION: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/group/subgroup/blob/*/catalog-info.yaml', +}; + +function setupFakeServer( + url: string, + callback: (request: { page: number; include_subgroups: boolean }) => { + data: GitLabProject[]; + nextPage?: number; + }, ) { server.use( - rest.get('https://gitlab.fake/api/v4/projects', (req, res, ctx) => { + rest.get(url, (req, res, ctx) => { if (req.headers.get('private-token') !== 'test-token') { return res(ctx.status(401), ctx.json({})); } const page = req.url.searchParams.get('page'); + const include_subgroups = req.url.searchParams.get('include_subgroups'); const response = callback({ page: parseInt(page!, 10), + include_subgroups: include_subgroups === 'true', }); // Filter the fake results based on the `last_activity_after` parameter @@ -97,15 +116,43 @@ describe('GitlabDiscoveryProcessor', () => { jest.useRealTimers(); }); - const location: LocationSpec = { - type: 'gitlab-discovery', - target: 'https://gitlab.fake/north-star', - }; + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect( + parseUrl('https://gitlab.com/group/subgroup/blob/master/catalog.yaml'), + ).toEqual({ + group: 'group/subgroup', + host: 'gitlab.com', + branch: 'master', + catalogPath: 'catalog.yaml', + }); + expect( + parseUrl('https://gitlab.com/blob/*/subfolder/catalog.yaml'), + ).toEqual({ + group: undefined, + host: 'gitlab.com', + branch: '*', + catalogPath: 'subfolder/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://gitlab.com')).toThrow(); + expect(() => parseUrl('https://gitlab.com//')).toThrow(); + expect(() => parseUrl('https://gitlab.com/foo')).toThrow(); + expect(() => parseUrl('https://gitlab.com//foo')).toThrow(); + expect(() => parseUrl('https://gitlab.com/org/teams')).toThrow(); + expect(() => parseUrl('https://gitlab.com/org//teams')).toThrow(); + expect(() => + parseUrl('https://gitlab.com/org//teams/blob/catalog.yaml'), + ).toThrow(); + }); + }); describe('handles repositories', () => { it('pages through all repositories', async () => { const processor = getProcessor(); - setupFakeGitLab(request => { + setupFakeServer(PROJECTS_URL, request => { switch (request.page) { case 1: return { @@ -145,7 +192,7 @@ describe('GitlabDiscoveryProcessor', () => { }); const result: any[] = []; - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result.push(e); }); expect(result).toEqual([ @@ -168,9 +215,78 @@ describe('GitlabDiscoveryProcessor', () => { ]); }); + it('can force a branch name', async () => { + const processor = getProcessor(); + setupFakeServer(PROJECTS_URL, request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(PROJECT_LOCATION_MASTER_BRANCH, false, e => { + result.push(e); + }); + expect(result).toEqual([ + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/1/-/blob/master/catalog-info.yaml', + }, + optional: true, + }, + ]); + }); + + it('can filter based on group', async () => { + const processor = getProcessor(); + setupFakeServer(GROUP_PROJECTS_URL, request => { + if (!request.include_subgroups) { + throw new Error('include_subgroups should be set'); + } + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(GROUP_LOCATION, false, e => { + result.push(e); + }); + // If everything was set up correctly, we should have received the fake repo specified above + expect(result).toHaveLength(1); + }); + it('uses the previous scan timestamp to filter', async () => { const processor = getProcessor(); - setupFakeGitLab(request => { + setupFakeServer(PROJECTS_URL, request => { switch (request.page) { case 1: return { @@ -199,7 +315,7 @@ describe('GitlabDiscoveryProcessor', () => { const result: any[] = []; // First scan should find all repos, since no last activity was cached - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result.push(e); }); expect(result).toHaveLength(2); @@ -207,7 +323,7 @@ describe('GitlabDiscoveryProcessor', () => { // Second scan should have used the mocked Date to set the last scanned time to 2001 // This should result in only the second repo being scanned, since that has a timestamp of 2002 const result2: any[] = []; - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result2.push(e); }); expect(result2).toHaveLength(1); @@ -217,7 +333,7 @@ describe('GitlabDiscoveryProcessor', () => { describe('handles failure', () => { it('invalid token', async () => { // Setup an empty fake gitlab, since we don't care about actual results - setupFakeGitLab(_ => { + setupFakeServer(PROJECTS_URL, _ => { return { data: [], }; @@ -226,7 +342,7 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); config.integrations.gitlab[0].token = 'invalid'; await expect( - getProcessor(config).readLocation(location, false, _ => {}), + getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/Unauthorized/); }); @@ -234,14 +350,14 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); delete config.integrations; await expect( - getProcessor(config).readLocation(location, false, _ => {}), + getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/no GitLab integration/); }); it('location type', async () => { const incorrectLocation: LocationSpec = { type: 'something-that-is-not-gitlab-discovery', - target: 'https://gitlab.fake/north-star', + target: 'https://gitlab.fake/oh-dear', }; await expect( diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index b3cce6e711..5603a1aa7f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -16,17 +16,19 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { - ScmIntegrations, -} from '@backstage/integration'; +import { ScmIntegrations } from '@backstage/integration'; import { Logger } from 'winston'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { GitLabClient, GitLabProject, paginated } from "./gitlab"; -import { CacheClient, CacheManager, PluginCacheManager } from "@backstage/backend-common"; +import { GitLabClient, GitLabProject, paginated } from './gitlab'; +import { + CacheClient, + CacheManager, + PluginCacheManager, +} from '@backstage/backend-common'; /** - * Extracts repositories out of a GitLab org. + * Extracts repositories out of an GitLab instance. */ export class GitLabDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; @@ -35,9 +37,8 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { static fromConfig(config: Config, options: { logger: Logger }) { const integrations = ScmIntegrations.fromConfig(config); - const pluginCache = CacheManager.fromConfig(config).forPlugin( - 'gitlab-discovery', - ); + const pluginCache = + CacheManager.fromConfig(config).forPlugin('gitlab-discovery'); return new GitLabDiscoveryProcessor({ ...options, @@ -65,10 +66,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { return false; } - const integration = this.integrations.gitlab.byUrl(location.target); + const { group, host, branch, catalogPath } = parseUrl(location.target); + + const integration = this.integrations.gitlab.byUrl(`https://${host}`); if (!integration) { throw new Error( - `There is no GitLab integration that matches ${location.target}. Please add a configuration entry for it under integrations.gitlab`, + `There is no GitLab integration that matches ${host}. Please add a configuration entry for it under integrations.gitlab`, ); } @@ -80,6 +83,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { this.logger.info(`Reading GitLab projects from ${location.target}`); const projects = paginated(options => client.listProjects(options), { + group, last_activity_after: await this.updateLastActivity(), page: 1, }); @@ -96,12 +100,19 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { } for (const project of result.matches) { + const project_branch = branch === '*' ? project.default_branch : branch; + emit( results.location( { type: 'url', - // The format expected by the GitLabUrlReader: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath - target: `${project.web_url}/-/blob/${project.default_branch}/catalog-info.yaml`, + // The format expected by the GitLabUrlReader: + // https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + // + // This unfortunately will trigger another API call in `getGitLabFileFetchUrl` to get the project ID. + // The alternative is using the `buildRawUrl` function, which does not support subgroups, so providing a raw + // URL here won't work either. + target: `${project.web_url}/-/blob/${project_branch}/${catalogPath}`, }, true, ), @@ -127,3 +138,37 @@ type Result = { scanned: number; matches: GitLabProject[]; }; + +/* + * Helpers + */ + +export function parseUrl(urlString: string): { + group?: string; + host: string; + branch: string; + catalogPath: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // (/group/subgroup)/blob/branch|*/filepath + const blobIndex = path.findIndex(p => p === 'blob'); + if (blobIndex !== -1 && path.length > blobIndex + 2) { + const group = + blobIndex > 0 ? path.slice(0, blobIndex).join('/') : undefined; + + return { + group, + host: url.host, + branch: decodeURIComponent(path[blobIndex + 1]), + catalogPath: decodeURIComponent(path.slice(blobIndex + 2).join('/')), + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +export function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts index fb7763dd1c..8781e071f7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -31,6 +31,18 @@ export class GitLabClient { } async listProjects(options?: ListOptions): Promise> { + if (options?.group) { + return this.pagedRequest( + `${this.config.apiBaseUrl}/groups/${encodeURIComponent( + options?.group, + )}/projects`, + { + ...options, + include_subgroups: true, + }, + ); + } + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); } @@ -69,7 +81,8 @@ export class GitLabClient { } export type ListOptions = { - [key: string]: string | number | undefined; + [key: string]: string | number | boolean | undefined; + group?: string; per_page?: number | undefined; page?: number | undefined; };