From 47930a36f79bc09890bdf9c1bd3dae3a7c01e074 Mon Sep 17 00:00:00 2001 From: mateiidavid Date: Thu, 10 Dec 2020 12:34:09 +0000 Subject: [PATCH] Implement readTree for GitlabReader (#3547) * This change adds an implementation for the readTree method in GitlabReader. The implementation is similar to the readTree method in GithubReader with the exception of the 'path' variable used for subpaths in the archive. To facilitate this, GitlabIntegrationConfig now has an apiUrl field which was missing for Gitlab but was present for Bitbucket & GH. * As part of this change, there are also new tests added to GitlabReader to mirror what has been done with the GH reader. For now the archive format chosen is 'zip', follow-up action includes having a look at whether tar.gz is preferable. Signed-off-by: Matei David --- .../src/reading/GitlabUrlReader.test.ts | 286 ++++++++++++------ .../src/reading/GitlabUrlReader.ts | 66 +++- packages/integration/src/gitlab/config.ts | 20 +- 3 files changed, 273 insertions(+), 99 deletions(-) diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 3a76c0631b..6cc5e30dbd 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -14,12 +14,14 @@ * limitations under the License. */ +import { ConfigReader } from '@backstage/config'; +import { msw } from '@backstage/test-utils'; +import fs from 'fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import { ConfigReader } from '@backstage/config'; +import path from 'path'; import { getVoidLogger } from '../logging'; import { GitlabUrlReader } from './GitlabUrlReader'; -import { msw } from '@backstage/test-utils'; import { ReadTreeResponseFactory } from './tree'; const logger = getVoidLogger(); @@ -30,105 +32,207 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ describe('GitlabUrlReader', () => { const worker = setupServer(); - msw.setupDefaultHandlers(worker); - beforeEach(() => { - worker.use( - rest.get('*/api/v4/projects/:name', (_, res, ctx) => - res(ctx.status(200), ctx.json({ id: 12345 })), - ), - rest.get('*', (req, res, ctx) => - res( - ctx.status(200), - ctx.json({ - url: req.url.toString(), - headers: req.headers.getAllHeaders(), - }), + describe('implementation', () => { + beforeEach(() => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), ), - ), - ); - }); - - const createConfig = (token?: string) => - new ConfigReader( - { - integrations: { gitlab: [{ host: 'gitlab.com', token }] }, - }, - 'test-config', - ); - - it.each([ - // Project URLs - { - url: - 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - config: createConfig(), - response: expect.objectContaining({ - url: - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - headers: expect.objectContaining({ - 'private-token': '', - }), - }), - }, - { - url: - 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - config: createConfig('0123456789'), - response: expect.objectContaining({ - url: - 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - headers: expect.objectContaining({ - 'private-token': '0123456789', - }), - }), - }, - { - url: - 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup - config: createConfig(), - response: expect.objectContaining({ - url: - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - }), - }, - - // Raw URLs - { - url: 'https://gitlab.example.com/a/b/blob/master/c.yaml', - config: createConfig(), - response: expect.objectContaining({ - url: 'https://gitlab.example.com/a/b/raw/master/c.yaml', - }), - }, - ])('should handle happy path %#', async ({ url, config, response }) => { - const [{ reader }] = GitlabUrlReader.factory({ - config, - logger, - treeResponseFactory, + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), + ), + ); }); - const data = await reader.read(url); - const res = await JSON.parse(data.toString('utf-8')); - expect(res).toEqual(response); - }); + const createConfig = (token?: string) => + new ConfigReader( + { + integrations: { gitlab: [{ host: 'gitlab.com', token }] }, + }, + 'test-config', + ); - it.each([ - { - url: '', - config: createConfig(''), - error: - "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string", - }, - ])('should handle error path %#', async ({ url, config, error }) => { - await expect(async () => { + it.each([ + // Project URLs + { + url: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + headers: expect.objectContaining({ + 'private-token': '', + }), + }), + }, + { + url: + 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig('0123456789'), + response: expect.objectContaining({ + url: + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + headers: expect.objectContaining({ + 'private-token': '0123456789', + }), + }), + }, + { + url: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + }), + }, + + // Raw URLs + { + url: 'https://gitlab.example.com/a/b/blob/master/c.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: 'https://gitlab.example.com/a/b/raw/master/c.yaml', + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { const [{ reader }] = GitlabUrlReader.factory({ config, logger, treeResponseFactory, }); - await reader.read(url); - }).rejects.toThrow(error); + + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); + }); + + it.each([ + { + url: '', + config: createConfig(''), + error: + "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = GitlabUrlReader.factory({ + config, + logger, + treeResponseFactory, + }); + await reader.read(url); + }).rejects.toThrow(error); + }); + }); + + describe('readTree', () => { + const repoBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'repo.zip'), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://gitlab.com/backstage/mock/-/archive/repo/mock-repo.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + }); + + it('returns the wanted files from an archive', async () => { + const processor = new GitlabUrlReader( + { host: 'gitlab.com' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://gitlab.com/backstage/mock/tree/repo', + ); + + const files = await response.files(); + expect(files.length).toBe(2); + + const indexMarkdownFile = await files[0].content(); + const mkDocsFile = await files[1].content(); + + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('returns the wanted files from hosted gitlab', async () => { + worker.use( + rest.get( + 'https://git.mycompany.com/backstage/mock/-/archive/repo/mock-repo.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + + const processor = new GitlabUrlReader( + { host: 'git.mycompany.com' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://git.mycompany.com/backstage/mock/tree/repo/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('throws an error when branch is not specified', async () => { + const processor = new GitlabUrlReader( + { host: 'gitlab.com' }, + { treeResponseFactory }, + ); + + await expect( + processor.readTree('https://gitlab.com/backstage/mock'), + ).rejects.toThrow( + 'GitLab URL must contain a branch to be able to fetch its tree', + ); + }); + + it('returns the wanted files from an archive with a subpath', async () => { + const processor = new GitlabUrlReader( + { host: 'gitlab.com' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://gitlab.com/backstage/mock/tree/repo/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); }); }); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index d6d1da5cbb..d51e8dc090 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -21,22 +21,37 @@ import { readGitLabIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; -import { NotFoundError } from '../errors'; -import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; +import { InputError, NotFoundError } from '../errors'; +import { ReadTreeResponseFactory } from './tree'; +import { + ReaderFactory, + ReadTreeOptions, + ReadTreeResponse, + UrlReader, +} from './types'; +import parseGitUri from 'git-url-parse'; +import { Readable } from 'stream'; export class GitlabUrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + private readonly treeResponseFactory: ReadTreeResponseFactory; + + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const configs = readGitLabIntegrationConfigs( config.getOptionalConfigArray('integrations.gitlab') ?? [], ); return configs.map(options => { - const reader = new GitlabUrlReader(options); + const reader = new GitlabUrlReader(options, { treeResponseFactory }); const predicate = (url: URL) => url.host === options.host; return { reader, predicate }; }); }; - constructor(private readonly options: GitLabIntegrationConfig) {} + constructor( + private readonly options: GitLabIntegrationConfig, + deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) { + this.treeResponseFactory = deps.treeResponseFactory; + } async read(url: string): Promise { const builtUrl = await getGitLabFileFetchUrl(url, this.options); @@ -59,8 +74,45 @@ export class GitlabUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { - throw new Error('GitlabUrlReader does not implement readTree'); + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const { + name: repoName, + ref, + protocol, + resource, + full_name, + filepath, + } = parseGitUri(url); + + if (!ref) { + throw new InputError( + 'GitLab URL must contain a branch to be able to fetch its tree', + ); + } + + const archive = `${protocol}://${resource}/${full_name}/-/archive/${ref}/${repoName}-${ref}.zip`; + const response = await fetch( + archive, + getGitLabRequestOptions(this.options), + ); + if (!response.ok) { + const msg = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(msg); + } + throw new Error(msg); + } + + const path = filepath ? `${repoName}-${ref}/${filepath}/` : ''; + + return this.treeResponseFactory.fromZipArchive({ + stream: (response.body as unknown) as Readable, + path, + filter: options?.filter, + }); } toString() { diff --git a/packages/integration/src/gitlab/config.ts b/packages/integration/src/gitlab/config.ts index a38312d01a..2d9f4b39ab 100644 --- a/packages/integration/src/gitlab/config.ts +++ b/packages/integration/src/gitlab/config.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; import { isValidHost } from '../helpers'; const GITLAB_HOST = 'gitlab.com'; +const GITLAB_API_BASE_URL = 'gitlab.com/api/v4'; /** * The configuration parameters for a single GitLab integration. @@ -28,6 +29,17 @@ export type GitLabIntegrationConfig = { */ host: string; + /** + * The base URL of the API of this provider, e.g. "https://gitlab.com/api/v4", + * with no trailing slash. + * + * May be omitted specifically for GitLab; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + /** * The authorization token to use for requests this provider. * @@ -45,6 +57,7 @@ export function readGitLabIntegrationConfig( config: Config, ): GitLabIntegrationConfig { const host = config.getOptionalString('host') ?? GITLAB_HOST; + let apiBaseUrl = config.getOptionalString('apiBaseUrl'); const token = config.getOptionalString('token'); if (!isValidHost(host)) { @@ -53,7 +66,12 @@ export function readGitLabIntegrationConfig( ); } - return { host, token }; + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (host === GITLAB_HOST) { + apiBaseUrl = GITLAB_API_BASE_URL; + } + return { host, token, apiBaseUrl }; } /**