Merge pull request #11463 from goququ/master

feat/catalog-backend-module-gitlab: create url location only if file…
2022-05-20 11:31:11 +02:00
parent 5f3dfe0d69 30824042f1
commit a91d2ee8e7
6 changed files with 196 additions and 24 deletions
@@ -0,0 +1,14 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': patch
+---
+
+do not create location object if file with component definition do not exists in project, that decrease count of request to gitlab with 404 status code. Now we can create processor with new flag to enable this logic:
+
+```ts
+const processor = GitLabDiscoveryProcessor.fromConfig(config, {
+  logger,
+  skipReposWithoutExactFileMatch: true,
+});
+```
+
+**WARNING:** This new functionality does not support globs in the repo file path
@@ -50,3 +50,5 @@ of your backend.
 +    GitLabDiscoveryProcessor.fromConfig(env.config, { logger: env.logger })
 +  );
 ```
+
+If you don't want create location object if file with component definition do not exists in project, you can set the `skipReposWithoutExactFileMatch` option. That can reduce count of request to gitlab with 404 status code.
@@ -16,6 +16,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
    config: Config,
    options: {
      logger: Logger;
+      skipReposWithoutExactFileMatch?: boolean;
    },
  ): GitLabDiscoveryProcessor;
  // (undocumented)
@@ -24,26 +24,37 @@ import { GitLabProject } from './lib';

 const server = setupServer();

-const PROJECTS_URL = 'https://gitlab.fake/api/v4/projects';
-const GROUP_PROJECTS_URL =
-  'https://gitlab.fake/api/v4/groups/group%2Fsubgroup/projects';
+const DOMAIN = 'gitlab.fake';
+const SERVER_URL = `https://${DOMAIN}`;
+const API_URL = `${SERVER_URL}/api/v4`;
+const PROJECTS_URL = `${API_URL}/projects`;
+const GROUP_PROJECTS_URL = `${API_URL}/groups/group%2Fsubgroup/projects`;
+const EXISTING_PROJECT_PATH = 'exist';

 const PROJECT_LOCATION: LocationSpec = {
  type: 'gitlab-discovery',
-  target: 'https://gitlab.fake/blob/*/catalog-info.yaml',
+  target: `${SERVER_URL}/blob/*/catalog-info.yaml`,
 };
 const PROJECT_LOCATION_MASTER_BRANCH: LocationSpec = {
  type: 'gitlab-discovery',
-  target: 'https://gitlab.fake/blob/master/catalog-info.yaml',
+  target: `${SERVER_URL}/blob/master/catalog-info.yaml`,
 };
 const GROUP_LOCATION: LocationSpec = {
  type: 'gitlab-discovery',
-  target: 'https://gitlab.fake/group/subgroup/blob/*/catalog-info.yaml',
+  target: `${SERVER_URL}/group/subgroup/blob/*/catalog-info.yaml`,
+};
+
+const GROUP_LOCATION_CUSTOM_BRANCH: LocationSpec = {
+  type: 'gitlab-discovery',
+  target: `${SERVER_URL}/group/subgroup/blob/test/catalog-info.yaml`,
 };

 function setupFakeServer(
  url: string,
-  callback: (request: { page: number; include_subgroups: boolean }) => {
+  listProjectsCallback: (request: {
+    page: number;
+    include_subgroups: boolean;
+  }) => {
    data: GitLabProject[];
    nextPage?: number;
  },
@@ -55,7 +66,7 @@ function setupFakeServer(
      }
      const page = req.url.searchParams.get('page');
      const include_subgroups = req.url.searchParams.get('include_subgroups');
-      const response = callback({
+      const response = listProjectsCallback({
        page: parseInt(page!, 10),
        include_subgroups: include_subgroups === 'true',
      });
@@ -75,6 +86,25 @@ function setupFakeServer(
        ctx.json(filteredData),
      );
    }),
+    rest.head(
+      `${API_URL}/projects/:project_path/repository/files/:file_path`,
+      (req, res, ctx) => {
+        if (req.headers.get('private-token') !== 'test-token') {
+          return res(ctx.status(401), ctx.json({}));
+        }
+        const ref = req.url.searchParams.get('ref');
+
+        if (ref === 'main' || ref === 'master') {
+          return res(ctx.status(200));
+        }
+
+        if (EXISTING_PROJECT_PATH === req.params.project_path) {
+          return res(ctx.status(200));
+        }
+
+        return res(ctx.status(404));
+      },
+    ),
  );
 }

@@ -86,8 +116,8 @@ function getConfig(): any {
    integrations: {
      gitlab: [
        {
-          host: 'gitlab.fake',
-          apiBaseUrl: 'https://gitlab.fake/api/v4',
+          host: DOMAIN,
+          apiBaseUrl: API_URL,
          token: 'test-token',
        },
      ],
@@ -95,11 +125,18 @@ function getConfig(): any {
  };
 }

-function getProcessor(config?: any): GitLabDiscoveryProcessor {
+function getProcessor({
+  config,
+  options,
+}: {
+  config?: any;
+  options?: Partial<Parameters<typeof GitLabDiscoveryProcessor.fromConfig>[1]>;
+} = {}): GitLabDiscoveryProcessor {
  return GitLabDiscoveryProcessor.fromConfig(
    new ConfigReader(config || getConfig()),
    {
      logger: getVoidLogger(),
+      ...options,
    },
  );
 }
@@ -163,6 +200,15 @@ describe('GitlabDiscoveryProcessor', () => {
                  default_branch: 'main',
                  last_activity_at: '2021-08-05T11:03:05.774Z',
                  web_url: 'https://gitlab.fake/1',
+                  path_with_namespace: '1',
+                },
+                {
+                  id: 2,
+                  archived: false,
+                  default_branch: 'main',
+                  last_activity_at: '2021-08-05T11:03:05.774Z',
+                  web_url: 'https://gitlab.fake/g/2',
+                  path_with_namespace: 'g/2',
                },
              ],
              nextPage: 2,
@@ -170,26 +216,29 @@ describe('GitlabDiscoveryProcessor', () => {
          case 2:
            return {
              data: [
-                {
-                  id: 2,
-                  archived: false,
-                  default_branch: 'master',
-                  last_activity_at: '2021-08-05T11:03:05.774Z',
-                  web_url: 'https://gitlab.fake/2',
-                },
                {
                  id: 3,
-                  archived: true, // ARCHIVED
+                  archived: false,
                  default_branch: 'master',
                  last_activity_at: '2021-08-05T11:03:05.774Z',
                  web_url: 'https://gitlab.fake/3',
+                  path_with_namespace: '3',
                },
                {
                  id: 4,
+                  archived: true, // ARCHIVED
+                  default_branch: 'master',
+                  last_activity_at: '2021-08-05T11:03:05.774Z',
+                  web_url: 'https://gitlab.fake/4',
+                  path_with_namespace: '4',
+                },
+                {
+                  id: 5,
                  archived: false,
                  default_branch: undefined, // MISSING DEFAULT BRANCH
                  last_activity_at: '2021-08-05T11:03:05.774Z',
-                  web_url: 'https://gitlab.fake/4',
+                  web_url: 'https://gitlab.fake/g/5',
+                  path_with_namespace: 'g/5',
                },
              ],
            };
@@ -215,7 +264,15 @@ describe('GitlabDiscoveryProcessor', () => {
          type: 'location',
          location: {
            type: 'url',
-            target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml',
+            target: 'https://gitlab.fake/g/2/-/blob/main/catalog-info.yaml',
+            presence: 'optional',
+          },
+        },
+        {
+          type: 'location',
+          location: {
+            type: 'url',
+            target: 'https://gitlab.fake/3/-/blob/master/catalog-info.yaml',
            presence: 'optional',
          },
        },
@@ -235,6 +292,7 @@ describe('GitlabDiscoveryProcessor', () => {
                  default_branch: 'main',
                  last_activity_at: '2021-08-05T11:03:05.774Z',
                  web_url: 'https://gitlab.fake/1',
+                  path_with_namespace: '1',
                },
              ],
            };
@@ -275,6 +333,7 @@ describe('GitlabDiscoveryProcessor', () => {
                  default_branch: 'main',
                  last_activity_at: '2021-08-05T11:03:05.774Z',
                  web_url: 'https://gitlab.fake/1',
+                  path_with_namespace: '1',
                },
              ],
            };
@@ -291,6 +350,49 @@ describe('GitlabDiscoveryProcessor', () => {
      expect(result).toHaveLength(1);
    });

+    it('can filter based on file existing', async () => {
+      const processor = getProcessor({
+        options: { skipReposWithoutExactFileMatch: true },
+      });
+      setupFakeServer(GROUP_PROJECTS_URL, request => {
+        if (!request.include_subgroups) {
+          throw new Error('include_subgroups should be set');
+        }
+        switch (request.page) {
+          case 1:
+            return {
+              data: [
+                {
+                  id: 1,
+                  archived: false,
+                  default_branch: 'main',
+                  last_activity_at: '2021-08-05T11:03:05.774Z',
+                  web_url: 'https://gitlab.fake/1',
+                  path_with_namespace: '1',
+                },
+                {
+                  id: 1,
+                  archived: false,
+                  default_branch: 'main',
+                  last_activity_at: '2021-08-05T11:03:05.774Z',
+                  web_url: `https://gitlab.fake/${EXISTING_PROJECT_PATH}`,
+                  path_with_namespace: EXISTING_PROJECT_PATH,
+                },
+              ],
+            };
+          default:
+            throw new Error('Invalid request');
+        }
+      });
+
+      const result: any[] = [];
+      await processor.readLocation(GROUP_LOCATION_CUSTOM_BRANCH, false, e => {
+        result.push(e);
+      });
+      // If everything was set up correctly, we should have received the fake repo specified above
+      expect(result).toHaveLength(1);
+    });
+
    it('uses the previous scan timestamp to filter', async () => {
      const processor = getProcessor();
      setupFakeServer(PROJECTS_URL, request => {
@@ -304,6 +406,7 @@ describe('GitlabDiscoveryProcessor', () => {
                  default_branch: 'main',
                  last_activity_at: '2000-01-01T00:00:00Z',
                  web_url: 'https://gitlab.fake/1',
+                  path_with_namespace: '1',
                },
                {
                  id: 2,
@@ -311,6 +414,7 @@ describe('GitlabDiscoveryProcessor', () => {
                  default_branch: 'main',
                  last_activity_at: '2002-01-01T00:00:00Z',
                  web_url: 'https://gitlab.fake/2',
+                  path_with_namespace: '2',
                },
              ],
            };
@@ -349,7 +453,7 @@ describe('GitlabDiscoveryProcessor', () => {
      const config = getConfig();
      config.integrations.gitlab[0].token = 'invalid';
      await expect(
-        getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}),
+        getProcessor({ config }).readLocation(PROJECT_LOCATION, false, _ => {}),
      ).rejects.toThrow(/Unauthorized/);
    });

@@ -357,7 +461,7 @@ describe('GitlabDiscoveryProcessor', () => {
      const config = getConfig();
      delete config.integrations;
      await expect(
-        getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}),
+        getProcessor({ config }).readLocation(PROJECT_LOCATION, false, _ => {}),
      ).rejects.toThrow(/no GitLab integration/);
    });

@@ -41,8 +41,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
  private readonly integrations: ScmIntegrationRegistry;
  private readonly logger: Logger;
  private readonly cache: CacheClient;
+  private readonly skipReposWithoutExactFileMatch: boolean;

-  static fromConfig(config: Config, options: { logger: Logger }) {
+  static fromConfig(
+    config: Config,
+    options: { logger: Logger; skipReposWithoutExactFileMatch?: boolean },
+  ): GitLabDiscoveryProcessor {
    const integrations = ScmIntegrations.fromConfig(config);
    const pluginCache =
      CacheManager.fromConfig(config).forPlugin('gitlab-discovery');
@@ -58,10 +62,13 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
    integrations: ScmIntegrationRegistry;
    pluginCache: PluginCacheManager;
    logger: Logger;
+    skipReposWithoutExactFileMatch?: boolean;
  }) {
    this.integrations = options.integrations;
    this.cache = options.pluginCache.getClient();
    this.logger = options.logger;
+    this.skipReposWithoutExactFileMatch =
+      options.skipReposWithoutExactFileMatch || false;
  }

  getProcessorName(): string {
@@ -114,6 +121,20 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
        continue;
      }

+      if (this.skipReposWithoutExactFileMatch) {
+        const project_branch = branch === '*' ? project.default_branch : branch;
+
+        const projectHasFile: boolean = await client.hasFile(
+          project.path_with_namespace,
+          project_branch,
+          catalogPath,
+        );
+
+        if (!projectHasFile) {
+          continue;
+        }
+      }
+
      res.matches.push(project);
    }

@@ -63,6 +63,36 @@ export class GitLabClient {
    return this.pagedRequest(`/projects`, options);
  }

+  async hasFile(
+    projectPath: string,
+    branch: string,
+    filePath: string,
+  ): Promise<boolean> {
+    const endpoint: string = `/projects/${encodeURIComponent(
+      projectPath,
+    )}/repository/files/${encodeURIComponent(filePath)}`;
+    const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
+    request.searchParams.append('ref', branch);
+
+    const response = await fetch(request.toString(), {
+      headers: getGitLabRequestOptions(this.config).headers,
+      method: 'HEAD',
+    });
+
+    if (!response.ok) {
+      if (response.status >= 500) {
+        this.logger.debug(
+          `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
+            response.status
+          } - ${response.statusText}`,
+        );
+      }
+      return false;
+    }
+
+    return true;
+  }
+
  /**
   * Performs a request against a given paginated GitLab endpoint.
   *