First part of GitLabDiscoveryProcessor (no target parsing yet)

Signed-off-by: Roy Jacobs <roy.jacobs@gmail.com>

plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts

@@ -0,0 +1,252 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { getVoidLogger } from '@backstage/backend-common';
+import { LocationSpec } from '@backstage/catalog-model';
+import {
+  GitLabDiscoveryProcessor,
+} from './GitLabDiscoveryProcessor';
+import { setupServer } from 'msw/node';
+import { rest } from 'msw';
+import { GitLabProject } from "./gitlab";
+
+const server = setupServer();
+
+function setupFakeGitLab(
+  callback: (request: {
+    page: number;
+  }) => { data: GitLabProject[]; nextPage?: number },
+) {
+  server.use(
+    rest.get('https://gitlab.fake/api/v4/projects', (req, res, ctx) => {
+      if (req.headers.get('private-token') !== 'test-token') {
+        return res(ctx.status(401), ctx.json({}));
+      }
+      const page = req.url.searchParams.get('page');
+      const response = callback({
+        page: parseInt(page!, 10),
+      });
+
+      // Filter the fake results based on the `last_activity_after` parameter
+      const last_activity_after = req.url.searchParams.get(
+        'last_activity_after',
+      );
+      const filteredData = response.data.filter(
+        v =>
+          !last_activity_after ||
+          Date.parse(v.last_activity_at) >= Date.parse(last_activity_after),
+      );
+
+      return res(
+        ctx.set('x-next-page', response.nextPage?.toString() ?? ''),
+        ctx.json(filteredData),
+      );
+    }),
+  );
+}
+
+function getConfig(): any {
+  return {
+    backend: {
+      cache: { store: 'memory' },
+    },
+    integrations: {
+      gitlab: [
+        {
+          host: 'gitlab.fake',
+          apiBaseUrl: 'https://gitlab.fake/api/v4',
+          token: 'test-token',
+        },
+      ],
+    },
+  };
+}
+
+function getProcessor(config?: any): GitLabDiscoveryProcessor {
+  return GitLabDiscoveryProcessor.fromConfig(
+    new ConfigReader(config || getConfig()),
+    {
+      logger: getVoidLogger(),
+    },
+  );
+}
+
+describe('GitlabDiscoveryProcessor', () => {
+  beforeAll(() => {
+    server.listen();
+    jest.useFakeTimers('modern');
+    jest.setSystemTime(new Date('2001-01-01T12:34:56Z'));
+  });
+  afterEach(() => server.resetHandlers());
+  afterAll(() => {
+    server.close();
+    jest.useRealTimers();
+  });
+
+  const location: LocationSpec = {
+    type: 'gitlab-discovery',
+    target: 'https://gitlab.fake/north-star',
+  };
+
+  describe('handles repositories', () => {
+    it('pages through all repositories', async () => {
+      const processor = getProcessor();
+      setupFakeGitLab(request => {
+        switch (request.page) {
+          case 1:
+            return {
+              data: [
+                {
+                  id: 1,
+                  archived: false,
+                  default_branch: 'main',
+                  last_activity_at: '2021-08-05T11:03:05.774Z',
+                  web_url: 'https://gitlab.fake/1',
+                },
+              ],
+              nextPage: 2,
+            };
+          case 2:
+            return {
+              data: [
+                {
+                  id: 2,
+                  archived: false,
+                  default_branch: 'master',
+                  last_activity_at: '2021-08-05T11:03:05.774Z',
+                  web_url: 'https://gitlab.fake/2',
+                },
+                {
+                  id: 3,
+                  archived: true, // ARCHIVED
+                  default_branch: 'master',
+                  last_activity_at: '2021-08-05T11:03:05.774Z',
+                  web_url: 'https://gitlab.fake/3',
+                },
+              ],
+            };
+          default:
+            throw new Error('Invalid request');
+        }
+      });
+
+      const result: any[] = [];
+      await processor.readLocation(location, false, e => {
+        result.push(e);
+      });
+      expect(result).toEqual([
+        {
+          type: 'location',
+          location: {
+            type: 'url',
+            target: 'https://gitlab.fake/1/-/blob/main/catalog-info.yaml',
+          },
+          optional: true,
+        },
+        {
+          type: 'location',
+          location: {
+            type: 'url',
+            target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml',
+          },
+          optional: true,
+        },
+      ]);
+    });
+
+    it('uses the previous scan timestamp to filter', async () => {
+      const processor = getProcessor();
+      setupFakeGitLab(request => {
+        switch (request.page) {
+          case 1:
+            return {
+              data: [
+                {
+                  id: 1,
+                  archived: false,
+                  default_branch: 'main',
+                  last_activity_at: '2000-01-01T00:00:00Z',
+                  web_url: 'https://gitlab.fake/1',
+                },
+                {
+                  id: 2,
+                  archived: false,
+                  default_branch: 'main',
+                  last_activity_at: '2002-01-01T00:00:00Z',
+                  web_url: 'https://gitlab.fake/2',
+                },
+              ],
+            };
+          default:
+            throw new Error('Invalid request');
+        }
+      });
+
+      const result: any[] = [];
+
+      // First scan should find all repos, since no last activity was cached
+      await processor.readLocation(location, false, e => {
+        result.push(e);
+      });
+      expect(result).toHaveLength(2);
+
+      // Second scan should have used the mocked Date to set the last scanned time to 2001
+      // This should result in only the second repo being scanned, since that has a timestamp of 2002
+      const result2: any[] = [];
+      await processor.readLocation(location, false, e => {
+        result2.push(e);
+      });
+      expect(result2).toHaveLength(1);
+    });
+  });
+
+  describe('handles failure', () => {
+    it('invalid token', async () => {
+      // Setup an empty fake gitlab, since we don't care about actual results
+      setupFakeGitLab(_ => {
+        return {
+          data: [],
+        };
+      });
+
+      const config = getConfig();
+      config.integrations.gitlab[0].token = 'invalid';
+      await expect(
+        getProcessor(config).readLocation(location, false, _ => {}),
+      ).rejects.toThrow(/Unauthorized/);
+    });
+
+    it('missing integration', async () => {
+      const config = getConfig();
+      delete config.integrations;
+      await expect(
+        getProcessor(config).readLocation(location, false, _ => {}),
+      ).rejects.toThrow(/no GitLab integration/);
+    });
+
+    it('location type', async () => {
+      const incorrectLocation: LocationSpec = {
+        type: 'something-that-is-not-gitlab-discovery',
+        target: 'https://gitlab.fake/north-star',
+      };
+
+      await expect(
+        getProcessor().readLocation(incorrectLocation, false, _ => {}),
+      ).resolves.toBeFalsy();
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import {
+  ScmIntegrations,
+} from '@backstage/integration';
+import { Logger } from 'winston';
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+import { GitLabClient, GitLabProject, paginated } from "./gitlab";
+import { CacheClient, CacheManager, PluginCacheManager } from "@backstage/backend-common";
+
+/**
+ * Extracts repositories out of a GitLab org.
+ */
+export class GitLabDiscoveryProcessor implements CatalogProcessor {
+  private readonly integrations: ScmIntegrations;
+  private readonly logger: Logger;
+  private readonly cache: CacheClient;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const integrations = ScmIntegrations.fromConfig(config);
+    const pluginCache = CacheManager.fromConfig(config).forPlugin(
+      'gitlab-discovery',
+    );
+
+    return new GitLabDiscoveryProcessor({
+      ...options,
+      integrations,
+      pluginCache,
+    });
+  }
+
+  private constructor(options: {
+    integrations: ScmIntegrations;
+    pluginCache: PluginCacheManager;
+    logger: Logger;
+  }) {
+    this.integrations = options.integrations;
+    this.cache = options.pluginCache.getClient();
+    this.logger = options.logger;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'gitlab-discovery') {
+      return false;
+    }
+
+    const integration = this.integrations.gitlab.byUrl(location.target);
+    if (!integration) {
+      throw new Error(
+        `There is no GitLab integration that matches ${location.target}. Please add a configuration entry for it under integrations.gitlab`,
+      );
+    }
+
+    const client = new GitLabClient({
+      config: integration.config,
+      logger: this.logger,
+    });
+    const startTimestamp = Date.now();
+    this.logger.info(`Reading GitLab projects from ${location.target}`);
+
+    const projects = paginated(options => client.listProjects(options), {
+      last_activity_after: await this.updateLastActivity(),
+      page: 1,
+    });
+
+    const result: Result = {
+      scanned: 0,
+      matches: [],
+    };
+    for await (const project of projects) {
+      result.scanned++;
+      if (!project.archived) {
+        result.matches.push(project);
+      }
+    }
+
+    for (const project of result.matches) {
+      emit(
+        results.location(
+          {
+            type: 'url',
+            // The format expected by the GitLabUrlReader: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
+            target: `${project.web_url}/-/blob/${project.default_branch}/catalog-info.yaml`,
+          },
+          true,
+        ),
+      );
+    }
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${result.scanned} GitLab repositories in ${duration} seconds`,
+    );
+
+    return true;
+  }
+
+  async updateLastActivity(): Promise<string | undefined> {
+    const lastActivity = await this.cache.get('last-activity');
+    await this.cache.set('last-activity', new Date().toISOString());
+    return lastActivity as string | undefined;
+  }
+}
+
+type Result = {
+  scanned: number;
+  matches: GitLabProject[];
+};

plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch from 'cross-fetch';
+import {
+  getGitLabRequestOptions,
+  GitLabIntegrationConfig,
+} from '@backstage/integration';
+import { Logger } from 'winston';
+
+export class GitLabClient {
+  private readonly config: GitLabIntegrationConfig;
+  private readonly logger: Logger;
+
+  constructor(options: { config: GitLabIntegrationConfig; logger: Logger }) {
+    this.config = options.config;
+    this.logger = options.logger;
+  }
+
+  async listProjects(options?: ListOptions): Promise<PagedResponse<any>> {
+    return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options);
+  }
+
+  private async pagedRequest(
+    endpoint: string,
+    options?: ListOptions,
+  ): Promise<PagedResponse<any>> {
+    const request = new URL(endpoint);
+    for (const key in options) {
+      if (options[key]) {
+        request.searchParams.append(key, options[key]!.toString());
+      }
+    }
+
+    this.logger.debug(`Fetching: ${request.toString()}`);
+    const response = await fetch(
+      request.toString(),
+      getGitLabRequestOptions(this.config),
+    );
+    if (!response.ok) {
+      throw new Error(
+        `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
+          response.status
+        } - ${response.statusText}`,
+      );
+    }
+    return response.json().then(items => {
+      const nextPage = response.headers.get('x-next-page');
+
+      return {
+        items,
+        nextPage: nextPage ? Number(nextPage) : null,
+      } as PagedResponse<any>;
+    });
+  }
+}
+
+export type ListOptions = {
+  [key: string]: string | number | undefined;
+  per_page?: number | undefined;
+  page?: number | undefined;
+};
+
+export type PagedResponse<T> = {
+  items: T[];
+  nextPage?: number;
+};
+
+export async function* paginated(
+  request: (options: ListOptions) => Promise<PagedResponse<any>>,
+  options: ListOptions,
+) {
+  let res;
+  do {
+    res = await request(options);
+    options.page = res.nextPage;
+    for (const item of res.items) {
+      yield item;
+    }
+  } while (res.nextPage);
+}

plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { GitLabClient, paginated } from './client';
+export type { GitLabProject } from "./types";

plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type GitLabProject = {
+  id: number;
+  default_branch: string;
+  archived: boolean;
+  last_activity_at: string;
+  web_url: string;
+};

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -26,6 +26,7 @@ export { FileReaderProcessor } from './FileReaderProcessor';
 export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
 export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
 export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor';
+export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor';
 export { LocationEntityProcessor } from './LocationEntityProcessor';
 export { PlaceholderProcessor } from './PlaceholderProcessor';
 export type { PlaceholderResolver } from './PlaceholderProcessor';

plugins/catalog-backend/src/next/NextCatalogBuilder.ts

@@ -50,6 +50,7 @@ import {
   FileReaderProcessor,
   GithubDiscoveryProcessor,
   GithubOrgReaderProcessor,
+  GitLabDiscoveryProcessor,
   PlaceholderProcessor,
   PlaceholderResolver,
   UrlReaderProcessor,
@@ -372,6 +373,7 @@ export class NextCatalogBuilder {
         BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
         GithubDiscoveryProcessor.fromConfig(config, { logger }),
         GithubOrgReaderProcessor.fromConfig(config, { logger }),
+        GitLabDiscoveryProcessor.fromConfig(config, { logger }),
         new UrlReaderProcessor({ reader, logger }),
         CodeOwnersProcessor.fromConfig(config, { logger, reader }),
         new AnnotateLocationEntityProcessor({ integrations }),

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -46,6 +46,7 @@ import {
   FileReaderProcessor,
   GithubDiscoveryProcessor,
   GithubOrgReaderProcessor,
+  GitLabDiscoveryProcessor,
   HigherOrderOperation,
   HigherOrderOperations,
   LocationEntityProcessor,
@@ -316,6 +317,7 @@ export class CatalogBuilder {
         BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
         GithubDiscoveryProcessor.fromConfig(config, { logger }),
         GithubOrgReaderProcessor.fromConfig(config, { logger }),
+        GitLabDiscoveryProcessor.fromConfig(config, { logger }),
         new UrlReaderProcessor({ reader, logger }),
         CodeOwnersProcessor.fromConfig(config, { logger, reader }),
         new LocationEntityProcessor({ integrations }),