test: add GitLabClient coverage in catalog-backend

Signed-off-by: Minn Soe <contributions@minn.io>

plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts

@@ -0,0 +1,238 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ConfigReader } from '@backstage/config';
+import { setupRequestMockHandlers } from '@backstage/test-utils';
+import { readGitLabIntegrationConfig } from '@backstage/integration';
+import { getVoidLogger } from '@backstage/backend-common';
+import { rest } from 'msw';
+import { setupServer, SetupServerApi } from 'msw/node';
+
+import { GitLabClient, paginated } from './client';
+
+const server = setupServer();
+setupRequestMockHandlers(server);
+
+const MOCK_CONFIG = readGitLabIntegrationConfig(
+  new ConfigReader({
+    host: 'example.com',
+    token: 'test-token',
+    apiBaseUrl: 'https://example.com/api/v4',
+  }),
+);
+
+const FAKE_PAGED_ENDPOINT = `${MOCK_CONFIG.apiBaseUrl}/some-endpoint`;
+
+function setupFakeFourPageEndpoint(srv: SetupServerApi, endpoint: string) {
+  srv.use(
+    rest.get(endpoint, (req, res, ctx) => {
+      const page = req.url.searchParams.get('page');
+      const currentPage = page ? Number(page) : 1;
+      const fakePageCount = 4;
+
+      return res(
+        ctx.set({
+          'x-next-page':
+            currentPage < fakePageCount ? String(currentPage + 1) : '',
+        }),
+        ctx.json([{ someContentOfPage: currentPage }]),
+      );
+    }),
+  );
+}
+
+function setupFakeGroupProjectsEndpoint(
+  srv: SetupServerApi,
+  apiBaseUrl: string,
+  groupID: string,
+) {
+  srv.use(
+    rest.get(`${apiBaseUrl}/groups/${groupID}/projects`, (_, res, ctx) => {
+      return res(
+        ctx.set('x-next-page', ''),
+        ctx.json([
+          {
+            id: 1,
+            description: 'Project One Description',
+            name: 'Project One',
+            path: 'project-one',
+          },
+        ]),
+      );
+    }),
+  );
+}
+
+function setupFakeInstanceProjectsEndpoint(
+  srv: SetupServerApi,
+  apiBaseUrl: string,
+) {
+  srv.use(
+    rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => {
+      return res(
+        ctx.set('x-next-page', ''),
+        ctx.json([
+          {
+            id: 1,
+            description: 'Project One Description',
+            name: 'Project One',
+            path: 'project-one',
+          },
+          {
+            id: 2,
+            description: 'Project Two Description',
+            name: 'Project Two',
+            path: 'project-two',
+          },
+        ]),
+      );
+    }),
+  );
+}
+
+describe('GitLabClient', () => {
+  describe('pagedRequest', () => {
+    beforeEach(() => {
+      // setup fake paginated endpoint with 4 pages each returning one item
+      setupFakeFourPageEndpoint(server, FAKE_PAGED_ENDPOINT);
+    });
+
+    it('should provide immediate items within the page', async () => {
+      const client = new GitLabClient({
+        config: MOCK_CONFIG,
+        logger: getVoidLogger(),
+      });
+
+      const { items } = await client.pagedRequest(FAKE_PAGED_ENDPOINT);
+      // fake page contains exactly one item
+      expect(items).toHaveLength(1);
+    });
+
+    it('should request items for a given page number', async () => {
+      const client = new GitLabClient({
+        config: MOCK_CONFIG,
+        logger: getVoidLogger(),
+      });
+
+      const requestedPage = 2;
+      const { items, nextPage } = await client.pagedRequest(
+        FAKE_PAGED_ENDPOINT,
+        {
+          page: requestedPage,
+        },
+      );
+      // should contain an item from a given page
+      expect(items[0].someContentOfPage).toEqual(requestedPage);
+      // should set the nextPage property to the next page
+      expect(nextPage).toEqual(3);
+    });
+
+    it('should not have a next page if at the end', async () => {
+      const client = new GitLabClient({
+        config: MOCK_CONFIG,
+        logger: getVoidLogger(),
+      });
+
+      const { items, nextPage } = await client.pagedRequest(
+        FAKE_PAGED_ENDPOINT,
+        {
+          page: 4,
+        },
+      );
+      // should contain item of last page
+      expect(items).toHaveLength(1);
+      expect(nextPage).toBeNull();
+    });
+
+    it('should throw if response is not okay', async () => {
+      const endpoint = `${MOCK_CONFIG.apiBaseUrl}/unhealthy-endpoint`;
+      server.use(
+        rest.get(endpoint, (_, res, ctx) => {
+          return res(ctx.status(400), ctx.json({ error: 'some error' }));
+        }),
+      );
+
+      const client = new GitLabClient({
+        config: MOCK_CONFIG,
+        logger: getVoidLogger(),
+      });
+      // non-200 status code should throw
+      await expect(() => client.pagedRequest(endpoint)).rejects.toThrowError();
+    });
+  });
+
+  describe('listProjects', () => {
+    it('should get projects for a given group', async () => {
+      setupFakeGroupProjectsEndpoint(
+        server,
+        MOCK_CONFIG.apiBaseUrl,
+        'test-group',
+      );
+      const client = new GitLabClient({
+        config: MOCK_CONFIG,
+        logger: getVoidLogger(),
+      });
+
+      const groupProjectsGen = paginated(
+        options => client.listProjects(options),
+        { group: 'test-group' },
+      );
+      const allItems = [];
+      for await (const item of groupProjectsGen) {
+        allItems.push(item);
+      }
+      expect(allItems).toHaveLength(1);
+    });
+
+    it('should get all projects for an instance', async () => {
+      setupFakeInstanceProjectsEndpoint(server, MOCK_CONFIG.apiBaseUrl);
+      const client = new GitLabClient({
+        config: MOCK_CONFIG,
+        logger: getVoidLogger(),
+      });
+
+      const instanceProjectsGen = paginated(
+        options => client.listProjects(options),
+        {},
+      );
+      const allItems = [];
+      for await (const item of instanceProjectsGen) {
+        allItems.push(item);
+      }
+      expect(allItems).toHaveLength(2);
+    });
+  });
+});
+
+describe('paginated', () => {
+  it('should iterate through the pages until exhausted', async () => {
+    setupFakeFourPageEndpoint(server, FAKE_PAGED_ENDPOINT);
+    const client = new GitLabClient({
+      config: MOCK_CONFIG,
+      logger: getVoidLogger(),
+    });
+
+    const paginatedAsyncGenerator = paginated(
+      options => client.pagedRequest(FAKE_PAGED_ENDPOINT, options),
+      {},
+    );
+    const allItems = [];
+    for await (const item of paginatedAsyncGenerator) {
+      allItems.push(item);
+    }
+
+    expect(allItems).toHaveLength(4);
+  });
+});

plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts

@@ -46,7 +46,19 @@ export class GitLabClient {
     return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options);
   }
 
-  private async pagedRequest(
+  /**
+   * Performs a request against a given paginated GitLab endpoint.
+   *
+   * This method may be used to perform authenticated REST calls against any
+   * paginated GitLab endpoint which uses X-NEXT-PAGE headers. The return value
+   * can be be used with the {@link paginated} async-generator function to yield
+   * each item from the paged request.
+   *
+   * @see {@link paginated}
+   * @param endpoint - The complete request URL for the endpoint.
+   * @param options - Request queryString options which may also include page variables.
+   */
+  async pagedRequest(
     endpoint: string,
     options?: ListOptions,
   ): Promise<PagedResponse<any>> {
@@ -92,6 +104,18 @@ export type PagedResponse<T> = {
   nextPage?: number;
 };
 
+/**
+ * Advances through each page and provides each item from a paginated request.
+ *
+ * The async generator function yields each item from repeated calls to the
+ * provided request function. The generator walks through each available page by
+ * setting the page key in the options passed into the request function and
+ * making repeated calls until there are no more pages.
+ *
+ * @see {@link pagedRequest}
+ * @param request - Function which returns a PagedResponse to walk through.
+ * @param options - Initial ListOptions for the request function.
+ */
 export async function* paginated(
   request: (options: ListOptions) => Promise<PagedResponse<any>>,
   options: ListOptions,