From af80fb7a013c32c99070a5ed77dd8f17bbbadad7 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Wed, 1 Dec 2021 19:58:52 +0000 Subject: [PATCH 1/5] test: add GitLabClient coverage in catalog-backend Signed-off-by: Minn Soe --- .../processors/gitlab/client.test.ts | 238 ++++++++++++++++++ .../src/ingestion/processors/gitlab/client.ts | 26 +- 2 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts new file mode 100644 index 0000000000..e2c919ae9b --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts @@ -0,0 +1,238 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ConfigReader } from '@backstage/config'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { readGitLabIntegrationConfig } from '@backstage/integration'; +import { getVoidLogger } from '@backstage/backend-common'; +import { rest } from 'msw'; +import { setupServer, SetupServerApi } from 'msw/node'; + +import { GitLabClient, paginated } from './client'; + +const server = setupServer(); +setupRequestMockHandlers(server); + +const MOCK_CONFIG = readGitLabIntegrationConfig( + new ConfigReader({ + host: 'example.com', + token: 'test-token', + apiBaseUrl: 'https://example.com/api/v4', + }), +); + +const FAKE_PAGED_ENDPOINT = `${MOCK_CONFIG.apiBaseUrl}/some-endpoint`; + +function setupFakeFourPageEndpoint(srv: SetupServerApi, endpoint: string) { + srv.use( + rest.get(endpoint, (req, res, ctx) => { + const page = req.url.searchParams.get('page'); + const currentPage = page ? Number(page) : 1; + const fakePageCount = 4; + + return res( + ctx.set({ + 'x-next-page': + currentPage < fakePageCount ? String(currentPage + 1) : '', + }), + ctx.json([{ someContentOfPage: currentPage }]), + ); + }), + ); +} + +function setupFakeGroupProjectsEndpoint( + srv: SetupServerApi, + apiBaseUrl: string, + groupID: string, +) { + srv.use( + rest.get(`${apiBaseUrl}/groups/${groupID}/projects`, (_, res, ctx) => { + return res( + ctx.set('x-next-page', ''), + ctx.json([ + { + id: 1, + description: 'Project One Description', + name: 'Project One', + path: 'project-one', + }, + ]), + ); + }), + ); +} + +function setupFakeInstanceProjectsEndpoint( + srv: SetupServerApi, + apiBaseUrl: string, +) { + srv.use( + rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => { + return res( + ctx.set('x-next-page', ''), + ctx.json([ + { + id: 1, + description: 'Project One Description', + name: 'Project One', + path: 'project-one', + }, + { + id: 2, + description: 'Project Two Description', + name: 'Project Two', + path: 'project-two', + }, + ]), + ); + }), + ); +} + +describe('GitLabClient', () => { + describe('pagedRequest', () => { + beforeEach(() => { + // setup fake paginated endpoint with 4 pages each returning one item + setupFakeFourPageEndpoint(server, FAKE_PAGED_ENDPOINT); + }); + + it('should provide immediate items within the page', async () => { + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const { items } = await client.pagedRequest(FAKE_PAGED_ENDPOINT); + // fake page contains exactly one item + expect(items).toHaveLength(1); + }); + + it('should request items for a given page number', async () => { + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const requestedPage = 2; + const { items, nextPage } = await client.pagedRequest( + FAKE_PAGED_ENDPOINT, + { + page: requestedPage, + }, + ); + // should contain an item from a given page + expect(items[0].someContentOfPage).toEqual(requestedPage); + // should set the nextPage property to the next page + expect(nextPage).toEqual(3); + }); + + it('should not have a next page if at the end', async () => { + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const { items, nextPage } = await client.pagedRequest( + FAKE_PAGED_ENDPOINT, + { + page: 4, + }, + ); + // should contain item of last page + expect(items).toHaveLength(1); + expect(nextPage).toBeNull(); + }); + + it('should throw if response is not okay', async () => { + const endpoint = `${MOCK_CONFIG.apiBaseUrl}/unhealthy-endpoint`; + server.use( + rest.get(endpoint, (_, res, ctx) => { + return res(ctx.status(400), ctx.json({ error: 'some error' })); + }), + ); + + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + // non-200 status code should throw + await expect(() => client.pagedRequest(endpoint)).rejects.toThrowError(); + }); + }); + + describe('listProjects', () => { + it('should get projects for a given group', async () => { + setupFakeGroupProjectsEndpoint( + server, + MOCK_CONFIG.apiBaseUrl, + 'test-group', + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const groupProjectsGen = paginated( + options => client.listProjects(options), + { group: 'test-group' }, + ); + const allItems = []; + for await (const item of groupProjectsGen) { + allItems.push(item); + } + expect(allItems).toHaveLength(1); + }); + + it('should get all projects for an instance', async () => { + setupFakeInstanceProjectsEndpoint(server, MOCK_CONFIG.apiBaseUrl); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const instanceProjectsGen = paginated( + options => client.listProjects(options), + {}, + ); + const allItems = []; + for await (const item of instanceProjectsGen) { + allItems.push(item); + } + expect(allItems).toHaveLength(2); + }); + }); +}); + +describe('paginated', () => { + it('should iterate through the pages until exhausted', async () => { + setupFakeFourPageEndpoint(server, FAKE_PAGED_ENDPOINT); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const paginatedAsyncGenerator = paginated( + options => client.pagedRequest(FAKE_PAGED_ENDPOINT, options), + {}, + ); + const allItems = []; + for await (const item of paginatedAsyncGenerator) { + allItems.push(item); + } + + expect(allItems).toHaveLength(4); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts index d1765521a3..75832883e4 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -46,7 +46,19 @@ export class GitLabClient { return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); } - private async pagedRequest( + /** + * Performs a request against a given paginated GitLab endpoint. + * + * This method may be used to perform authenticated REST calls against any + * paginated GitLab endpoint which uses X-NEXT-PAGE headers. The return value + * can be be used with the {@link paginated} async-generator function to yield + * each item from the paged request. + * + * @see {@link paginated} + * @param endpoint - The complete request URL for the endpoint. + * @param options - Request queryString options which may also include page variables. + */ + async pagedRequest( endpoint: string, options?: ListOptions, ): Promise> { @@ -92,6 +104,18 @@ export type PagedResponse = { nextPage?: number; }; +/** + * Advances through each page and provides each item from a paginated request. + * + * The async generator function yields each item from repeated calls to the + * provided request function. The generator walks through each available page by + * setting the page key in the options passed into the request function and + * making repeated calls until there are no more pages. + * + * @see {@link pagedRequest} + * @param request - Function which returns a PagedResponse to walk through. + * @param options - Initial ListOptions for the request function. + */ export async function* paginated( request: (options: ListOptions) => Promise>, options: ListOptions, From d58e92a7f064a8a0c76eaf98fd35d727975cfd40 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Tue, 7 Dec 2021 13:46:12 +0000 Subject: [PATCH 2/5] feat: allow types in gitlab client paginated reqs Signed-off-by: Minn Soe --- .../src/ingestion/processors/gitlab/client.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts index 75832883e4..fbaa816d92 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -58,10 +58,10 @@ export class GitLabClient { * @param endpoint - The complete request URL for the endpoint. * @param options - Request queryString options which may also include page variables. */ - async pagedRequest( + async pagedRequest( endpoint: string, options?: ListOptions, - ): Promise> { + ): Promise> { const request = new URL(endpoint); for (const key in options) { if (options[key]) { @@ -116,8 +116,8 @@ export type PagedResponse = { * @param request - Function which returns a PagedResponse to walk through. * @param options - Initial ListOptions for the request function. */ -export async function* paginated( - request: (options: ListOptions) => Promise>, +export async function* paginated( + request: (options: ListOptions) => Promise>, options: ListOptions, ) { let res; From 91b3ad3553a43576a352d7c9598d834845736eb5 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Fri, 3 Dec 2021 13:32:01 +0000 Subject: [PATCH 3/5] refactor: simplify by using GitLab apiBaseUrl Signed-off-by: Minn Soe --- .../processors/gitlab/client.test.ts | 40 ++++++++++--------- .../src/ingestion/processors/gitlab/client.ts | 34 ++++++++-------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts index e2c919ae9b..92ee65e5e7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts @@ -32,21 +32,22 @@ const MOCK_CONFIG = readGitLabIntegrationConfig( apiBaseUrl: 'https://example.com/api/v4', }), ); +const FAKE_PAGED_ENDPOINT = `/some-endpoint`; +const FAKE_PAGED_URL = `${MOCK_CONFIG.apiBaseUrl}${FAKE_PAGED_ENDPOINT}`; -const FAKE_PAGED_ENDPOINT = `${MOCK_CONFIG.apiBaseUrl}/some-endpoint`; - -function setupFakeFourPageEndpoint(srv: SetupServerApi, endpoint: string) { +function setupFakeFourPageURL(srv: SetupServerApi, url: string) { srv.use( - rest.get(endpoint, (req, res, ctx) => { + rest.get(url, (req, res, ctx) => { const page = req.url.searchParams.get('page'); const currentPage = page ? Number(page) : 1; const fakePageCount = 4; return res( - ctx.set({ - 'x-next-page': - currentPage < fakePageCount ? String(currentPage + 1) : '', - }), + // set next page number header if page requested is less than count + ctx.set( + 'x-next-page', + currentPage < fakePageCount ? String(currentPage + 1) : '', + ), ctx.json([{ someContentOfPage: currentPage }]), ); }), @@ -106,7 +107,7 @@ describe('GitLabClient', () => { describe('pagedRequest', () => { beforeEach(() => { // setup fake paginated endpoint with 4 pages each returning one item - setupFakeFourPageEndpoint(server, FAKE_PAGED_ENDPOINT); + setupFakeFourPageURL(server, FAKE_PAGED_URL); }); it('should provide immediate items within the page', async () => { @@ -157,9 +158,10 @@ describe('GitLabClient', () => { }); it('should throw if response is not okay', async () => { - const endpoint = `${MOCK_CONFIG.apiBaseUrl}/unhealthy-endpoint`; + const endpoint = '/unhealthy-endpoint'; + const url = `${MOCK_CONFIG.apiBaseUrl}${endpoint}`; server.use( - rest.get(endpoint, (_, res, ctx) => { + rest.get(url, (_, res, ctx) => { return res(ctx.status(400), ctx.json({ error: 'some error' })); }), ); @@ -203,33 +205,33 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const instanceProjectsGen = paginated( + const instanceProjects = paginated( options => client.listProjects(options), {}, ); - const allItems = []; - for await (const item of instanceProjectsGen) { - allItems.push(item); + const allProjects = []; + for await (const project of instanceProjects) { + allProjects.push(project); } - expect(allItems).toHaveLength(2); + expect(allProjects).toHaveLength(2); }); }); }); describe('paginated', () => { it('should iterate through the pages until exhausted', async () => { - setupFakeFourPageEndpoint(server, FAKE_PAGED_ENDPOINT); + setupFakeFourPageURL(server, FAKE_PAGED_URL); const client = new GitLabClient({ config: MOCK_CONFIG, logger: getVoidLogger(), }); - const paginatedAsyncGenerator = paginated( + const paginatedItems = paginated( options => client.pagedRequest(FAKE_PAGED_ENDPOINT, options), {}, ); const allItems = []; - for await (const item of paginatedAsyncGenerator) { + for await (const item of paginatedItems) { allItems.push(item); } diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts index fbaa816d92..1df5ce962f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -21,6 +21,18 @@ import { } from '@backstage/integration'; import { Logger } from 'winston'; +export type ListOptions = { + [key: string]: string | number | boolean | undefined; + group?: string; + per_page?: number | undefined; + page?: number | undefined; +}; + +export type PagedResponse = { + items: T[]; + nextPage?: number; +}; + export class GitLabClient { private readonly config: GitLabIntegrationConfig; private readonly logger: Logger; @@ -33,9 +45,7 @@ export class GitLabClient { async listProjects(options?: ListOptions): Promise> { if (options?.group) { return this.pagedRequest( - `${this.config.apiBaseUrl}/groups/${encodeURIComponent( - options?.group, - )}/projects`, + `/groups/${encodeURIComponent(options?.group)}/projects`, { ...options, include_subgroups: true, @@ -43,7 +53,7 @@ export class GitLabClient { ); } - return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); + return this.pagedRequest(`/projects`, options); } /** @@ -55,14 +65,14 @@ export class GitLabClient { * each item from the paged request. * * @see {@link paginated} - * @param endpoint - The complete request URL for the endpoint. + * @param endpoint - The request endpoint, e.g. /projects. * @param options - Request queryString options which may also include page variables. */ async pagedRequest( endpoint: string, options?: ListOptions, ): Promise> { - const request = new URL(endpoint); + const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); for (const key in options) { if (options[key]) { request.searchParams.append(key, options[key]!.toString()); @@ -92,18 +102,6 @@ export class GitLabClient { } } -export type ListOptions = { - [key: string]: string | number | boolean | undefined; - group?: string; - per_page?: number | undefined; - page?: number | undefined; -}; - -export type PagedResponse = { - items: T[]; - nextPage?: number; -}; - /** * Advances through each page and provides each item from a paginated request. * From b61a5bbdf320ac2cb02506b55566f0704056f1fb Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Tue, 7 Dec 2021 13:36:28 +0000 Subject: [PATCH 4/5] feat: add gitlab self managed check method Signed-off-by: Minn Soe --- .../processors/gitlab/client.test.ts | 28 +++++++++++++++++++ .../src/ingestion/processors/gitlab/client.ts | 7 +++++ 2 files changed, 35 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts index 92ee65e5e7..b249137740 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts @@ -104,6 +104,34 @@ function setupFakeInstanceProjectsEndpoint( } describe('GitLabClient', () => { + describe('isSelfManaged', () => { + it('returns true if self managed instance', () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader({ + host: 'example.com', + token: 'test-token', + apiBaseUrl: 'https://example.com/api/v4', + }), + ), + logger: getVoidLogger(), + }); + expect(client.isSelfManaged()).toBeTruthy(); + }); + it('returns false if gitlab.com', () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader({ + host: 'gitlab.com', + token: 'test-token', + }), + ), + logger: getVoidLogger(), + }); + expect(client.isSelfManaged()).toBeFalsy(); + }); + }); + describe('pagedRequest', () => { beforeEach(() => { // setup fake paginated endpoint with 4 pages each returning one item diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts index 1df5ce962f..4d42f5e068 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -42,6 +42,13 @@ export class GitLabClient { this.logger = options.logger; } + /** + * Indicates whether the client is for a SaaS or self managed GitLab instance. + */ + isSelfManaged(): boolean { + return this.config.host !== 'gitlab.com'; + } + async listProjects(options?: ListOptions): Promise> { if (options?.group) { return this.pagedRequest( From 6bccc7d794ebec402256de35021f95d7fea3b664 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Wed, 8 Dec 2021 13:21:53 +0000 Subject: [PATCH 5/5] changesets: add gitlab changes in catalog-backend Signed-off-by: Minn Soe --- .changeset/hip-bananas-laugh.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/hip-bananas-laugh.md diff --git a/.changeset/hip-bananas-laugh.md b/.changeset/hip-bananas-laugh.md new file mode 100644 index 0000000000..c3f09da880 --- /dev/null +++ b/.changeset/hip-bananas-laugh.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +The `pagedRequest` method in the GitLab ingestion client is now public for re-use and may be used to make other calls to the GitLab API. Developers can now pass in a type into the GitLab `paginated` and `pagedRequest` functions as generics instead of forcing `any` (defaults to `any` to maintain compatibility). The `GitLabClient` now provides a `isSelfManaged` convenience method.