From 47930a36f79bc09890bdf9c1bd3dae3a7c01e074 Mon Sep 17 00:00:00 2001 From: mateiidavid Date: Thu, 10 Dec 2020 12:34:09 +0000 Subject: [PATCH 1/4] Implement readTree for GitlabReader (#3547) * This change adds an implementation for the readTree method in GitlabReader. The implementation is similar to the readTree method in GithubReader with the exception of the 'path' variable used for subpaths in the archive. To facilitate this, GitlabIntegrationConfig now has an apiUrl field which was missing for Gitlab but was present for Bitbucket & GH. * As part of this change, there are also new tests added to GitlabReader to mirror what has been done with the GH reader. For now the archive format chosen is 'zip', follow-up action includes having a look at whether tar.gz is preferable. Signed-off-by: Matei David --- .../src/reading/GitlabUrlReader.test.ts | 286 ++++++++++++------ .../src/reading/GitlabUrlReader.ts | 66 +++- packages/integration/src/gitlab/config.ts | 20 +- 3 files changed, 273 insertions(+), 99 deletions(-) diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 3a76c0631b..6cc5e30dbd 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -14,12 +14,14 @@ * limitations under the License. */ +import { ConfigReader } from '@backstage/config'; +import { msw } from '@backstage/test-utils'; +import fs from 'fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import { ConfigReader } from '@backstage/config'; +import path from 'path'; import { getVoidLogger } from '../logging'; import { GitlabUrlReader } from './GitlabUrlReader'; -import { msw } from '@backstage/test-utils'; import { ReadTreeResponseFactory } from './tree'; const logger = getVoidLogger(); @@ -30,105 +32,207 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ describe('GitlabUrlReader', () => { const worker = setupServer(); - msw.setupDefaultHandlers(worker); - beforeEach(() => { - worker.use( - rest.get('*/api/v4/projects/:name', (_, res, ctx) => - res(ctx.status(200), ctx.json({ id: 12345 })), - ), - rest.get('*', (req, res, ctx) => - res( - ctx.status(200), - ctx.json({ - url: req.url.toString(), - headers: req.headers.getAllHeaders(), - }), + describe('implementation', () => { + beforeEach(() => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), ), - ), - ); - }); - - const createConfig = (token?: string) => - new ConfigReader( - { - integrations: { gitlab: [{ host: 'gitlab.com', token }] }, - }, - 'test-config', - ); - - it.each([ - // Project URLs - { - url: - 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - config: createConfig(), - response: expect.objectContaining({ - url: - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - headers: expect.objectContaining({ - 'private-token': '', - }), - }), - }, - { - url: - 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - config: createConfig('0123456789'), - response: expect.objectContaining({ - url: - 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - headers: expect.objectContaining({ - 'private-token': '0123456789', - }), - }), - }, - { - url: - 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup - config: createConfig(), - response: expect.objectContaining({ - url: - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - }), - }, - - // Raw URLs - { - url: 'https://gitlab.example.com/a/b/blob/master/c.yaml', - config: createConfig(), - response: expect.objectContaining({ - url: 'https://gitlab.example.com/a/b/raw/master/c.yaml', - }), - }, - ])('should handle happy path %#', async ({ url, config, response }) => { - const [{ reader }] = GitlabUrlReader.factory({ - config, - logger, - treeResponseFactory, + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), + ), + ); }); - const data = await reader.read(url); - const res = await JSON.parse(data.toString('utf-8')); - expect(res).toEqual(response); - }); + const createConfig = (token?: string) => + new ConfigReader( + { + integrations: { gitlab: [{ host: 'gitlab.com', token }] }, + }, + 'test-config', + ); - it.each([ - { - url: '', - config: createConfig(''), - error: - "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string", - }, - ])('should handle error path %#', async ({ url, config, error }) => { - await expect(async () => { + it.each([ + // Project URLs + { + url: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + headers: expect.objectContaining({ + 'private-token': '', + }), + }), + }, + { + url: + 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig('0123456789'), + response: expect.objectContaining({ + url: + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + headers: expect.objectContaining({ + 'private-token': '0123456789', + }), + }), + }, + { + url: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + }), + }, + + // Raw URLs + { + url: 'https://gitlab.example.com/a/b/blob/master/c.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: 'https://gitlab.example.com/a/b/raw/master/c.yaml', + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { const [{ reader }] = GitlabUrlReader.factory({ config, logger, treeResponseFactory, }); - await reader.read(url); - }).rejects.toThrow(error); + + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); + }); + + it.each([ + { + url: '', + config: createConfig(''), + error: + "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = GitlabUrlReader.factory({ + config, + logger, + treeResponseFactory, + }); + await reader.read(url); + }).rejects.toThrow(error); + }); + }); + + describe('readTree', () => { + const repoBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'repo.zip'), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://gitlab.com/backstage/mock/-/archive/repo/mock-repo.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + }); + + it('returns the wanted files from an archive', async () => { + const processor = new GitlabUrlReader( + { host: 'gitlab.com' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://gitlab.com/backstage/mock/tree/repo', + ); + + const files = await response.files(); + expect(files.length).toBe(2); + + const indexMarkdownFile = await files[0].content(); + const mkDocsFile = await files[1].content(); + + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('returns the wanted files from hosted gitlab', async () => { + worker.use( + rest.get( + 'https://git.mycompany.com/backstage/mock/-/archive/repo/mock-repo.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + + const processor = new GitlabUrlReader( + { host: 'git.mycompany.com' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://git.mycompany.com/backstage/mock/tree/repo/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('throws an error when branch is not specified', async () => { + const processor = new GitlabUrlReader( + { host: 'gitlab.com' }, + { treeResponseFactory }, + ); + + await expect( + processor.readTree('https://gitlab.com/backstage/mock'), + ).rejects.toThrow( + 'GitLab URL must contain a branch to be able to fetch its tree', + ); + }); + + it('returns the wanted files from an archive with a subpath', async () => { + const processor = new GitlabUrlReader( + { host: 'gitlab.com' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://gitlab.com/backstage/mock/tree/repo/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); }); }); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index d6d1da5cbb..d51e8dc090 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -21,22 +21,37 @@ import { readGitLabIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; -import { NotFoundError } from '../errors'; -import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; +import { InputError, NotFoundError } from '../errors'; +import { ReadTreeResponseFactory } from './tree'; +import { + ReaderFactory, + ReadTreeOptions, + ReadTreeResponse, + UrlReader, +} from './types'; +import parseGitUri from 'git-url-parse'; +import { Readable } from 'stream'; export class GitlabUrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + private readonly treeResponseFactory: ReadTreeResponseFactory; + + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const configs = readGitLabIntegrationConfigs( config.getOptionalConfigArray('integrations.gitlab') ?? [], ); return configs.map(options => { - const reader = new GitlabUrlReader(options); + const reader = new GitlabUrlReader(options, { treeResponseFactory }); const predicate = (url: URL) => url.host === options.host; return { reader, predicate }; }); }; - constructor(private readonly options: GitLabIntegrationConfig) {} + constructor( + private readonly options: GitLabIntegrationConfig, + deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) { + this.treeResponseFactory = deps.treeResponseFactory; + } async read(url: string): Promise { const builtUrl = await getGitLabFileFetchUrl(url, this.options); @@ -59,8 +74,45 @@ export class GitlabUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { - throw new Error('GitlabUrlReader does not implement readTree'); + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const { + name: repoName, + ref, + protocol, + resource, + full_name, + filepath, + } = parseGitUri(url); + + if (!ref) { + throw new InputError( + 'GitLab URL must contain a branch to be able to fetch its tree', + ); + } + + const archive = `${protocol}://${resource}/${full_name}/-/archive/${ref}/${repoName}-${ref}.zip`; + const response = await fetch( + archive, + getGitLabRequestOptions(this.options), + ); + if (!response.ok) { + const msg = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(msg); + } + throw new Error(msg); + } + + const path = filepath ? `${repoName}-${ref}/${filepath}/` : ''; + + return this.treeResponseFactory.fromZipArchive({ + stream: (response.body as unknown) as Readable, + path, + filter: options?.filter, + }); } toString() { diff --git a/packages/integration/src/gitlab/config.ts b/packages/integration/src/gitlab/config.ts index a38312d01a..2d9f4b39ab 100644 --- a/packages/integration/src/gitlab/config.ts +++ b/packages/integration/src/gitlab/config.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; import { isValidHost } from '../helpers'; const GITLAB_HOST = 'gitlab.com'; +const GITLAB_API_BASE_URL = 'gitlab.com/api/v4'; /** * The configuration parameters for a single GitLab integration. @@ -28,6 +29,17 @@ export type GitLabIntegrationConfig = { */ host: string; + /** + * The base URL of the API of this provider, e.g. "https://gitlab.com/api/v4", + * with no trailing slash. + * + * May be omitted specifically for GitLab; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + /** * The authorization token to use for requests this provider. * @@ -45,6 +57,7 @@ export function readGitLabIntegrationConfig( config: Config, ): GitLabIntegrationConfig { const host = config.getOptionalString('host') ?? GITLAB_HOST; + let apiBaseUrl = config.getOptionalString('apiBaseUrl'); const token = config.getOptionalString('token'); if (!isValidHost(host)) { @@ -53,7 +66,12 @@ export function readGitLabIntegrationConfig( ); } - return { host, token }; + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (host === GITLAB_HOST) { + apiBaseUrl = GITLAB_API_BASE_URL; + } + return { host, token, apiBaseUrl }; } /** From 19f5c5e9905178ee19f44ddab6e6c2302176531a Mon Sep 17 00:00:00 2001 From: mateiidavid Date: Tue, 15 Dec 2020 12:56:39 +0000 Subject: [PATCH 2/4] Change Gitlab config test to include apiBaseurl --- packages/integration/src/gitlab/config.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/integration/src/gitlab/config.test.ts b/packages/integration/src/gitlab/config.test.ts index 2c4224a2c4..e817465b83 100644 --- a/packages/integration/src/gitlab/config.test.ts +++ b/packages/integration/src/gitlab/config.test.ts @@ -41,7 +41,10 @@ describe('readGitLabIntegrationConfig', () => { it('inserts the defaults if missing', () => { const output = readGitLabIntegrationConfig(buildConfig({})); - expect(output).toEqual({ host: 'gitlab.com' }); + expect(output).toEqual({ + host: 'gitlab.com', + apiBaseUrl: 'gitlab.com/api/v4', + }); }); it('rejects funky configs', () => { From 4eafdec4a05aedb48fc69473014b86b1048d3107 Mon Sep 17 00:00:00 2001 From: mateiidavid Date: Wed, 16 Dec 2020 14:17:37 +0000 Subject: [PATCH 3/4] Create changeset --- .changeset/pink-ravens-destroy.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/pink-ravens-destroy.md diff --git a/.changeset/pink-ravens-destroy.md b/.changeset/pink-ravens-destroy.md new file mode 100644 index 0000000000..84a2c4a964 --- /dev/null +++ b/.changeset/pink-ravens-destroy.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': minor +'@backstage/integration': minor +--- + +Introduce readTree method for GitLab URL Reader From d8e9b75dc7cafa5357bb727b48178a2e82385d99 Mon Sep 17 00:00:00 2001 From: mateiidavid Date: Wed, 16 Dec 2020 16:46:59 +0000 Subject: [PATCH 4/4] Change changeset to patch --- .changeset/pink-ravens-destroy.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/pink-ravens-destroy.md b/.changeset/pink-ravens-destroy.md index 84a2c4a964..647ccd9f37 100644 --- a/.changeset/pink-ravens-destroy.md +++ b/.changeset/pink-ravens-destroy.md @@ -1,6 +1,6 @@ --- -'@backstage/backend-common': minor -'@backstage/integration': minor +'@backstage/backend-common': patch +'@backstage/integration': patch --- Introduce readTree method for GitLab URL Reader