diff --git a/.changeset/real-beers-type.md b/.changeset/real-beers-type.md new file mode 100644 index 0000000000..f09aafe007 --- /dev/null +++ b/.changeset/real-beers-type.md @@ -0,0 +1,14 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +do not create location object if file with component definition do not exists in project, that decrease count of request to gitlab with 404 status code. Now we can create processor with new flag to enable this logic: + +```ts +const processor = GitLabDiscoveryProcessor.fromConfig(config, { + logger, + skipReposWithoutExactFileMatch: true, +}); +``` + +**WARNING:** This new functionality does not support globs in the repo file path diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 8d805fa362..3715b89920 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -50,3 +50,5 @@ of your backend. + GitLabDiscoveryProcessor.fromConfig(env.config, { logger: env.logger }) + ); ``` + +If you don't want create location object if file with component definition do not exists in project, you can set the `skipReposWithoutExactFileMatch` option. That can reduce count of request to gitlab with 404 status code. diff --git a/plugins/catalog-backend-module-gitlab/api-report.md b/plugins/catalog-backend-module-gitlab/api-report.md index ec10f80939..63e6fe7ccb 100644 --- a/plugins/catalog-backend-module-gitlab/api-report.md +++ b/plugins/catalog-backend-module-gitlab/api-report.md @@ -16,6 +16,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { config: Config, options: { logger: Logger; + skipReposWithoutExactFileMatch?: boolean; }, ): GitLabDiscoveryProcessor; // (undocumented) diff --git a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts index dc4ffb7251..278efe8036 100644 --- a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts @@ -24,26 +24,37 @@ import { GitLabProject } from './lib'; const server = setupServer(); -const PROJECTS_URL = 'https://gitlab.fake/api/v4/projects'; -const GROUP_PROJECTS_URL = - 'https://gitlab.fake/api/v4/groups/group%2Fsubgroup/projects'; +const DOMAIN = 'gitlab.fake'; +const SERVER_URL = `https://${DOMAIN}`; +const API_URL = `${SERVER_URL}/api/v4`; +const PROJECTS_URL = `${API_URL}/projects`; +const GROUP_PROJECTS_URL = `${API_URL}/groups/group%2Fsubgroup/projects`; +const EXISTING_PROJECT_PATH = 'exist'; const PROJECT_LOCATION: LocationSpec = { type: 'gitlab-discovery', - target: 'https://gitlab.fake/blob/*/catalog-info.yaml', + target: `${SERVER_URL}/blob/*/catalog-info.yaml`, }; const PROJECT_LOCATION_MASTER_BRANCH: LocationSpec = { type: 'gitlab-discovery', - target: 'https://gitlab.fake/blob/master/catalog-info.yaml', + target: `${SERVER_URL}/blob/master/catalog-info.yaml`, }; const GROUP_LOCATION: LocationSpec = { type: 'gitlab-discovery', - target: 'https://gitlab.fake/group/subgroup/blob/*/catalog-info.yaml', + target: `${SERVER_URL}/group/subgroup/blob/*/catalog-info.yaml`, +}; + +const GROUP_LOCATION_CUSTOM_BRANCH: LocationSpec = { + type: 'gitlab-discovery', + target: `${SERVER_URL}/group/subgroup/blob/test/catalog-info.yaml`, }; function setupFakeServer( url: string, - callback: (request: { page: number; include_subgroups: boolean }) => { + listProjectsCallback: (request: { + page: number; + include_subgroups: boolean; + }) => { data: GitLabProject[]; nextPage?: number; }, @@ -55,7 +66,7 @@ function setupFakeServer( } const page = req.url.searchParams.get('page'); const include_subgroups = req.url.searchParams.get('include_subgroups'); - const response = callback({ + const response = listProjectsCallback({ page: parseInt(page!, 10), include_subgroups: include_subgroups === 'true', }); @@ -75,6 +86,25 @@ function setupFakeServer( ctx.json(filteredData), ); }), + rest.head( + `${API_URL}/projects/:project_path/repository/files/:file_path`, + (req, res, ctx) => { + if (req.headers.get('private-token') !== 'test-token') { + return res(ctx.status(401), ctx.json({})); + } + const ref = req.url.searchParams.get('ref'); + + if (ref === 'main' || ref === 'master') { + return res(ctx.status(200)); + } + + if (EXISTING_PROJECT_PATH === req.params.project_path) { + return res(ctx.status(200)); + } + + return res(ctx.status(404)); + }, + ), ); } @@ -86,8 +116,8 @@ function getConfig(): any { integrations: { gitlab: [ { - host: 'gitlab.fake', - apiBaseUrl: 'https://gitlab.fake/api/v4', + host: DOMAIN, + apiBaseUrl: API_URL, token: 'test-token', }, ], @@ -95,11 +125,18 @@ function getConfig(): any { }; } -function getProcessor(config?: any): GitLabDiscoveryProcessor { +function getProcessor({ + config, + options, +}: { + config?: any; + options?: Partial[1]>; +} = {}): GitLabDiscoveryProcessor { return GitLabDiscoveryProcessor.fromConfig( new ConfigReader(config || getConfig()), { logger: getVoidLogger(), + ...options, }, ); } @@ -163,6 +200,15 @@ describe('GitlabDiscoveryProcessor', () => { default_branch: 'main', last_activity_at: '2021-08-05T11:03:05.774Z', web_url: 'https://gitlab.fake/1', + path_with_namespace: '1', + }, + { + id: 2, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/g/2', + path_with_namespace: 'g/2', }, ], nextPage: 2, @@ -170,26 +216,29 @@ describe('GitlabDiscoveryProcessor', () => { case 2: return { data: [ - { - id: 2, - archived: false, - default_branch: 'master', - last_activity_at: '2021-08-05T11:03:05.774Z', - web_url: 'https://gitlab.fake/2', - }, { id: 3, - archived: true, // ARCHIVED + archived: false, default_branch: 'master', last_activity_at: '2021-08-05T11:03:05.774Z', web_url: 'https://gitlab.fake/3', + path_with_namespace: '3', }, { id: 4, + archived: true, // ARCHIVED + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/4', + path_with_namespace: '4', + }, + { + id: 5, archived: false, default_branch: undefined, // MISSING DEFAULT BRANCH last_activity_at: '2021-08-05T11:03:05.774Z', - web_url: 'https://gitlab.fake/4', + web_url: 'https://gitlab.fake/g/5', + path_with_namespace: 'g/5', }, ], }; @@ -215,7 +264,15 @@ describe('GitlabDiscoveryProcessor', () => { type: 'location', location: { type: 'url', - target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml', + target: 'https://gitlab.fake/g/2/-/blob/main/catalog-info.yaml', + presence: 'optional', + }, + }, + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/3/-/blob/master/catalog-info.yaml', presence: 'optional', }, }, @@ -235,6 +292,7 @@ describe('GitlabDiscoveryProcessor', () => { default_branch: 'main', last_activity_at: '2021-08-05T11:03:05.774Z', web_url: 'https://gitlab.fake/1', + path_with_namespace: '1', }, ], }; @@ -275,6 +333,7 @@ describe('GitlabDiscoveryProcessor', () => { default_branch: 'main', last_activity_at: '2021-08-05T11:03:05.774Z', web_url: 'https://gitlab.fake/1', + path_with_namespace: '1', }, ], }; @@ -291,6 +350,49 @@ describe('GitlabDiscoveryProcessor', () => { expect(result).toHaveLength(1); }); + it('can filter based on file existing', async () => { + const processor = getProcessor({ + options: { skipReposWithoutExactFileMatch: true }, + }); + setupFakeServer(GROUP_PROJECTS_URL, request => { + if (!request.include_subgroups) { + throw new Error('include_subgroups should be set'); + } + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + path_with_namespace: '1', + }, + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: `https://gitlab.fake/${EXISTING_PROJECT_PATH}`, + path_with_namespace: EXISTING_PROJECT_PATH, + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(GROUP_LOCATION_CUSTOM_BRANCH, false, e => { + result.push(e); + }); + // If everything was set up correctly, we should have received the fake repo specified above + expect(result).toHaveLength(1); + }); + it('uses the previous scan timestamp to filter', async () => { const processor = getProcessor(); setupFakeServer(PROJECTS_URL, request => { @@ -304,6 +406,7 @@ describe('GitlabDiscoveryProcessor', () => { default_branch: 'main', last_activity_at: '2000-01-01T00:00:00Z', web_url: 'https://gitlab.fake/1', + path_with_namespace: '1', }, { id: 2, @@ -311,6 +414,7 @@ describe('GitlabDiscoveryProcessor', () => { default_branch: 'main', last_activity_at: '2002-01-01T00:00:00Z', web_url: 'https://gitlab.fake/2', + path_with_namespace: '2', }, ], }; @@ -349,7 +453,7 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); config.integrations.gitlab[0].token = 'invalid'; await expect( - getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), + getProcessor({ config }).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/Unauthorized/); }); @@ -357,7 +461,7 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); delete config.integrations; await expect( - getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), + getProcessor({ config }).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/no GitLab integration/); }); diff --git a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts index 01de1f12a0..73b8c52358 100644 --- a/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts @@ -41,8 +41,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrationRegistry; private readonly logger: Logger; private readonly cache: CacheClient; + private readonly skipReposWithoutExactFileMatch: boolean; - static fromConfig(config: Config, options: { logger: Logger }) { + static fromConfig( + config: Config, + options: { logger: Logger; skipReposWithoutExactFileMatch?: boolean }, + ): GitLabDiscoveryProcessor { const integrations = ScmIntegrations.fromConfig(config); const pluginCache = CacheManager.fromConfig(config).forPlugin('gitlab-discovery'); @@ -58,10 +62,13 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { integrations: ScmIntegrationRegistry; pluginCache: PluginCacheManager; logger: Logger; + skipReposWithoutExactFileMatch?: boolean; }) { this.integrations = options.integrations; this.cache = options.pluginCache.getClient(); this.logger = options.logger; + this.skipReposWithoutExactFileMatch = + options.skipReposWithoutExactFileMatch || false; } getProcessorName(): string { @@ -114,6 +121,20 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { continue; } + if (this.skipReposWithoutExactFileMatch) { + const project_branch = branch === '*' ? project.default_branch : branch; + + const projectHasFile: boolean = await client.hasFile( + project.path_with_namespace, + project_branch, + catalogPath, + ); + + if (!projectHasFile) { + continue; + } + } + res.matches.push(project); } diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 4d42f5e068..27214f3be8 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -63,6 +63,36 @@ export class GitLabClient { return this.pagedRequest(`/projects`, options); } + async hasFile( + projectPath: string, + branch: string, + filePath: string, + ): Promise { + const endpoint: string = `/projects/${encodeURIComponent( + projectPath, + )}/repository/files/${encodeURIComponent(filePath)}`; + const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); + request.searchParams.append('ref', branch); + + const response = await fetch(request.toString(), { + headers: getGitLabRequestOptions(this.config).headers, + method: 'HEAD', + }); + + if (!response.ok) { + if (response.status >= 500) { + this.logger.debug( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return false; + } + + return true; + } + /** * Performs a request against a given paginated GitLab endpoint. *