diff --git a/.changeset/hip-bananas-laugh.md b/.changeset/hip-bananas-laugh.md new file mode 100644 index 0000000000..c3f09da880 --- /dev/null +++ b/.changeset/hip-bananas-laugh.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +The `pagedRequest` method in the GitLab ingestion client is now public for re-use and may be used to make other calls to the GitLab API. Developers can now pass in a type into the GitLab `paginated` and `pagedRequest` functions as generics instead of forcing `any` (defaults to `any` to maintain compatibility). The `GitLabClient` now provides a `isSelfManaged` convenience method. diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts new file mode 100644 index 0000000000..b249137740 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts @@ -0,0 +1,268 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ConfigReader } from '@backstage/config'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { readGitLabIntegrationConfig } from '@backstage/integration'; +import { getVoidLogger } from '@backstage/backend-common'; +import { rest } from 'msw'; +import { setupServer, SetupServerApi } from 'msw/node'; + +import { GitLabClient, paginated } from './client'; + +const server = setupServer(); +setupRequestMockHandlers(server); + +const MOCK_CONFIG = readGitLabIntegrationConfig( + new ConfigReader({ + host: 'example.com', + token: 'test-token', + apiBaseUrl: 'https://example.com/api/v4', + }), +); +const FAKE_PAGED_ENDPOINT = `/some-endpoint`; +const FAKE_PAGED_URL = `${MOCK_CONFIG.apiBaseUrl}${FAKE_PAGED_ENDPOINT}`; + +function setupFakeFourPageURL(srv: SetupServerApi, url: string) { + srv.use( + rest.get(url, (req, res, ctx) => { + const page = req.url.searchParams.get('page'); + const currentPage = page ? Number(page) : 1; + const fakePageCount = 4; + + return res( + // set next page number header if page requested is less than count + ctx.set( + 'x-next-page', + currentPage < fakePageCount ? String(currentPage + 1) : '', + ), + ctx.json([{ someContentOfPage: currentPage }]), + ); + }), + ); +} + +function setupFakeGroupProjectsEndpoint( + srv: SetupServerApi, + apiBaseUrl: string, + groupID: string, +) { + srv.use( + rest.get(`${apiBaseUrl}/groups/${groupID}/projects`, (_, res, ctx) => { + return res( + ctx.set('x-next-page', ''), + ctx.json([ + { + id: 1, + description: 'Project One Description', + name: 'Project One', + path: 'project-one', + }, + ]), + ); + }), + ); +} + +function setupFakeInstanceProjectsEndpoint( + srv: SetupServerApi, + apiBaseUrl: string, +) { + srv.use( + rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => { + return res( + ctx.set('x-next-page', ''), + ctx.json([ + { + id: 1, + description: 'Project One Description', + name: 'Project One', + path: 'project-one', + }, + { + id: 2, + description: 'Project Two Description', + name: 'Project Two', + path: 'project-two', + }, + ]), + ); + }), + ); +} + +describe('GitLabClient', () => { + describe('isSelfManaged', () => { + it('returns true if self managed instance', () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader({ + host: 'example.com', + token: 'test-token', + apiBaseUrl: 'https://example.com/api/v4', + }), + ), + logger: getVoidLogger(), + }); + expect(client.isSelfManaged()).toBeTruthy(); + }); + it('returns false if gitlab.com', () => { + const client = new GitLabClient({ + config: readGitLabIntegrationConfig( + new ConfigReader({ + host: 'gitlab.com', + token: 'test-token', + }), + ), + logger: getVoidLogger(), + }); + expect(client.isSelfManaged()).toBeFalsy(); + }); + }); + + describe('pagedRequest', () => { + beforeEach(() => { + // setup fake paginated endpoint with 4 pages each returning one item + setupFakeFourPageURL(server, FAKE_PAGED_URL); + }); + + it('should provide immediate items within the page', async () => { + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const { items } = await client.pagedRequest(FAKE_PAGED_ENDPOINT); + // fake page contains exactly one item + expect(items).toHaveLength(1); + }); + + it('should request items for a given page number', async () => { + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const requestedPage = 2; + const { items, nextPage } = await client.pagedRequest( + FAKE_PAGED_ENDPOINT, + { + page: requestedPage, + }, + ); + // should contain an item from a given page + expect(items[0].someContentOfPage).toEqual(requestedPage); + // should set the nextPage property to the next page + expect(nextPage).toEqual(3); + }); + + it('should not have a next page if at the end', async () => { + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const { items, nextPage } = await client.pagedRequest( + FAKE_PAGED_ENDPOINT, + { + page: 4, + }, + ); + // should contain item of last page + expect(items).toHaveLength(1); + expect(nextPage).toBeNull(); + }); + + it('should throw if response is not okay', async () => { + const endpoint = '/unhealthy-endpoint'; + const url = `${MOCK_CONFIG.apiBaseUrl}${endpoint}`; + server.use( + rest.get(url, (_, res, ctx) => { + return res(ctx.status(400), ctx.json({ error: 'some error' })); + }), + ); + + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + // non-200 status code should throw + await expect(() => client.pagedRequest(endpoint)).rejects.toThrowError(); + }); + }); + + describe('listProjects', () => { + it('should get projects for a given group', async () => { + setupFakeGroupProjectsEndpoint( + server, + MOCK_CONFIG.apiBaseUrl, + 'test-group', + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const groupProjectsGen = paginated( + options => client.listProjects(options), + { group: 'test-group' }, + ); + const allItems = []; + for await (const item of groupProjectsGen) { + allItems.push(item); + } + expect(allItems).toHaveLength(1); + }); + + it('should get all projects for an instance', async () => { + setupFakeInstanceProjectsEndpoint(server, MOCK_CONFIG.apiBaseUrl); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const instanceProjects = paginated( + options => client.listProjects(options), + {}, + ); + const allProjects = []; + for await (const project of instanceProjects) { + allProjects.push(project); + } + expect(allProjects).toHaveLength(2); + }); + }); +}); + +describe('paginated', () => { + it('should iterate through the pages until exhausted', async () => { + setupFakeFourPageURL(server, FAKE_PAGED_URL); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const paginatedItems = paginated( + options => client.pagedRequest(FAKE_PAGED_ENDPOINT, options), + {}, + ); + const allItems = []; + for await (const item of paginatedItems) { + allItems.push(item); + } + + expect(allItems).toHaveLength(4); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts index d1765521a3..4d42f5e068 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -21,6 +21,18 @@ import { } from '@backstage/integration'; import { Logger } from 'winston'; +export type ListOptions = { + [key: string]: string | number | boolean | undefined; + group?: string; + per_page?: number | undefined; + page?: number | undefined; +}; + +export type PagedResponse = { + items: T[]; + nextPage?: number; +}; + export class GitLabClient { private readonly config: GitLabIntegrationConfig; private readonly logger: Logger; @@ -30,12 +42,17 @@ export class GitLabClient { this.logger = options.logger; } + /** + * Indicates whether the client is for a SaaS or self managed GitLab instance. + */ + isSelfManaged(): boolean { + return this.config.host !== 'gitlab.com'; + } + async listProjects(options?: ListOptions): Promise> { if (options?.group) { return this.pagedRequest( - `${this.config.apiBaseUrl}/groups/${encodeURIComponent( - options?.group, - )}/projects`, + `/groups/${encodeURIComponent(options?.group)}/projects`, { ...options, include_subgroups: true, @@ -43,14 +60,26 @@ export class GitLabClient { ); } - return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); + return this.pagedRequest(`/projects`, options); } - private async pagedRequest( + /** + * Performs a request against a given paginated GitLab endpoint. + * + * This method may be used to perform authenticated REST calls against any + * paginated GitLab endpoint which uses X-NEXT-PAGE headers. The return value + * can be be used with the {@link paginated} async-generator function to yield + * each item from the paged request. + * + * @see {@link paginated} + * @param endpoint - The request endpoint, e.g. /projects. + * @param options - Request queryString options which may also include page variables. + */ + async pagedRequest( endpoint: string, options?: ListOptions, - ): Promise> { - const request = new URL(endpoint); + ): Promise> { + const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); for (const key in options) { if (options[key]) { request.searchParams.append(key, options[key]!.toString()); @@ -80,20 +109,20 @@ export class GitLabClient { } } -export type ListOptions = { - [key: string]: string | number | boolean | undefined; - group?: string; - per_page?: number | undefined; - page?: number | undefined; -}; - -export type PagedResponse = { - items: T[]; - nextPage?: number; -}; - -export async function* paginated( - request: (options: ListOptions) => Promise>, +/** + * Advances through each page and provides each item from a paginated request. + * + * The async generator function yields each item from repeated calls to the + * provided request function. The generator walks through each available page by + * setting the page key in the options passed into the request function and + * making repeated calls until there are no more pages. + * + * @see {@link pagedRequest} + * @param request - Function which returns a PagedResponse to walk through. + * @param options - Initial ListOptions for the request function. + */ +export async function* paginated( + request: (options: ListOptions) => Promise>, options: ListOptions, ) { let res;