diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 3a76c0631b..6cc5e30dbd 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -14,12 +14,14 @@ * limitations under the License. */ +import { ConfigReader } from '@backstage/config'; +import { msw } from '@backstage/test-utils'; +import fs from 'fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import { ConfigReader } from '@backstage/config'; +import path from 'path'; import { getVoidLogger } from '../logging'; import { GitlabUrlReader } from './GitlabUrlReader'; -import { msw } from '@backstage/test-utils'; import { ReadTreeResponseFactory } from './tree'; const logger = getVoidLogger(); @@ -30,105 +32,207 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ describe('GitlabUrlReader', () => { const worker = setupServer(); - msw.setupDefaultHandlers(worker); - beforeEach(() => { - worker.use( - rest.get('*/api/v4/projects/:name', (_, res, ctx) => - res(ctx.status(200), ctx.json({ id: 12345 })), - ), - rest.get('*', (req, res, ctx) => - res( - ctx.status(200), - ctx.json({ - url: req.url.toString(), - headers: req.headers.getAllHeaders(), - }), + describe('implementation', () => { + beforeEach(() => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), ), - ), - ); - }); - - const createConfig = (token?: string) => - new ConfigReader( - { - integrations: { gitlab: [{ host: 'gitlab.com', token }] }, - }, - 'test-config', - ); - - it.each([ - // Project URLs - { - url: - 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - config: createConfig(), - response: expect.objectContaining({ - url: - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - headers: expect.objectContaining({ - 'private-token': '', - }), - }), - }, - { - url: - 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - config: createConfig('0123456789'), - response: expect.objectContaining({ - url: - 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - headers: expect.objectContaining({ - 'private-token': '0123456789', - }), - }), - }, - { - url: - 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup - config: createConfig(), - response: expect.objectContaining({ - url: - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - }), - }, - - // Raw URLs - { - url: 'https://gitlab.example.com/a/b/blob/master/c.yaml', - config: createConfig(), - response: expect.objectContaining({ - url: 'https://gitlab.example.com/a/b/raw/master/c.yaml', - }), - }, - ])('should handle happy path %#', async ({ url, config, response }) => { - const [{ reader }] = GitlabUrlReader.factory({ - config, - logger, - treeResponseFactory, + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), + ), + ); }); - const data = await reader.read(url); - const res = await JSON.parse(data.toString('utf-8')); - expect(res).toEqual(response); - }); + const createConfig = (token?: string) => + new ConfigReader( + { + integrations: { gitlab: [{ host: 'gitlab.com', token }] }, + }, + 'test-config', + ); - it.each([ - { - url: '', - config: createConfig(''), - error: - "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string", - }, - ])('should handle error path %#', async ({ url, config, error }) => { - await expect(async () => { + it.each([ + // Project URLs + { + url: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + headers: expect.objectContaining({ + 'private-token': '', + }), + }), + }, + { + url: + 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig('0123456789'), + response: expect.objectContaining({ + url: + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + headers: expect.objectContaining({ + 'private-token': '0123456789', + }), + }), + }, + { + url: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + }), + }, + + // Raw URLs + { + url: 'https://gitlab.example.com/a/b/blob/master/c.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: 'https://gitlab.example.com/a/b/raw/master/c.yaml', + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { const [{ reader }] = GitlabUrlReader.factory({ config, logger, treeResponseFactory, }); - await reader.read(url); - }).rejects.toThrow(error); + + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); + }); + + it.each([ + { + url: '', + config: createConfig(''), + error: + "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = GitlabUrlReader.factory({ + config, + logger, + treeResponseFactory, + }); + await reader.read(url); + }).rejects.toThrow(error); + }); + }); + + describe('readTree', () => { + const repoBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'repo.zip'), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://gitlab.com/backstage/mock/-/archive/repo/mock-repo.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + }); + + it('returns the wanted files from an archive', async () => { + const processor = new GitlabUrlReader( + { host: 'gitlab.com' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://gitlab.com/backstage/mock/tree/repo', + ); + + const files = await response.files(); + expect(files.length).toBe(2); + + const indexMarkdownFile = await files[0].content(); + const mkDocsFile = await files[1].content(); + + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('returns the wanted files from hosted gitlab', async () => { + worker.use( + rest.get( + 'https://git.mycompany.com/backstage/mock/-/archive/repo/mock-repo.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + + const processor = new GitlabUrlReader( + { host: 'git.mycompany.com' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://git.mycompany.com/backstage/mock/tree/repo/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('throws an error when branch is not specified', async () => { + const processor = new GitlabUrlReader( + { host: 'gitlab.com' }, + { treeResponseFactory }, + ); + + await expect( + processor.readTree('https://gitlab.com/backstage/mock'), + ).rejects.toThrow( + 'GitLab URL must contain a branch to be able to fetch its tree', + ); + }); + + it('returns the wanted files from an archive with a subpath', async () => { + const processor = new GitlabUrlReader( + { host: 'gitlab.com' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://gitlab.com/backstage/mock/tree/repo/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); }); }); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index d6d1da5cbb..d51e8dc090 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -21,22 +21,37 @@ import { readGitLabIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; -import { NotFoundError } from '../errors'; -import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; +import { InputError, NotFoundError } from '../errors'; +import { ReadTreeResponseFactory } from './tree'; +import { + ReaderFactory, + ReadTreeOptions, + ReadTreeResponse, + UrlReader, +} from './types'; +import parseGitUri from 'git-url-parse'; +import { Readable } from 'stream'; export class GitlabUrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + private readonly treeResponseFactory: ReadTreeResponseFactory; + + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const configs = readGitLabIntegrationConfigs( config.getOptionalConfigArray('integrations.gitlab') ?? [], ); return configs.map(options => { - const reader = new GitlabUrlReader(options); + const reader = new GitlabUrlReader(options, { treeResponseFactory }); const predicate = (url: URL) => url.host === options.host; return { reader, predicate }; }); }; - constructor(private readonly options: GitLabIntegrationConfig) {} + constructor( + private readonly options: GitLabIntegrationConfig, + deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) { + this.treeResponseFactory = deps.treeResponseFactory; + } async read(url: string): Promise { const builtUrl = await getGitLabFileFetchUrl(url, this.options); @@ -59,8 +74,45 @@ export class GitlabUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { - throw new Error('GitlabUrlReader does not implement readTree'); + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const { + name: repoName, + ref, + protocol, + resource, + full_name, + filepath, + } = parseGitUri(url); + + if (!ref) { + throw new InputError( + 'GitLab URL must contain a branch to be able to fetch its tree', + ); + } + + const archive = `${protocol}://${resource}/${full_name}/-/archive/${ref}/${repoName}-${ref}.zip`; + const response = await fetch( + archive, + getGitLabRequestOptions(this.options), + ); + if (!response.ok) { + const msg = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(msg); + } + throw new Error(msg); + } + + const path = filepath ? `${repoName}-${ref}/${filepath}/` : ''; + + return this.treeResponseFactory.fromZipArchive({ + stream: (response.body as unknown) as Readable, + path, + filter: options?.filter, + }); } toString() { diff --git a/packages/integration/src/gitlab/config.ts b/packages/integration/src/gitlab/config.ts index a38312d01a..2d9f4b39ab 100644 --- a/packages/integration/src/gitlab/config.ts +++ b/packages/integration/src/gitlab/config.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; import { isValidHost } from '../helpers'; const GITLAB_HOST = 'gitlab.com'; +const GITLAB_API_BASE_URL = 'gitlab.com/api/v4'; /** * The configuration parameters for a single GitLab integration. @@ -28,6 +29,17 @@ export type GitLabIntegrationConfig = { */ host: string; + /** + * The base URL of the API of this provider, e.g. "https://gitlab.com/api/v4", + * with no trailing slash. + * + * May be omitted specifically for GitLab; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + /** * The authorization token to use for requests this provider. * @@ -45,6 +57,7 @@ export function readGitLabIntegrationConfig( config: Config, ): GitLabIntegrationConfig { const host = config.getOptionalString('host') ?? GITLAB_HOST; + let apiBaseUrl = config.getOptionalString('apiBaseUrl'); const token = config.getOptionalString('token'); if (!isValidHost(host)) { @@ -53,7 +66,12 @@ export function readGitLabIntegrationConfig( ); } - return { host, token }; + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (host === GITLAB_HOST) { + apiBaseUrl = GITLAB_API_BASE_URL; + } + return { host, token, apiBaseUrl }; } /**