diff --git a/.changeset/light-sloths-report.md b/.changeset/light-sloths-report.md new file mode 100644 index 0000000000..8e9c20e9a4 --- /dev/null +++ b/.changeset/light-sloths-report.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-defaults': patch +--- + +The `GerritUrlReader` can now read content from a commit and not only from the top of a branch. The +Gitiles URL must contain the full commit `SHA` hash like: `https://gerrit.com/gitiles/repo/+/2846e8dc327ae2f60249983b1c3b96f42f205bae/catalog-info.yaml`. diff --git a/.changeset/old-waves-camp.md b/.changeset/old-waves-camp.md new file mode 100644 index 0000000000..04e3b520ae --- /dev/null +++ b/.changeset/old-waves-camp.md @@ -0,0 +1,13 @@ +--- +'@backstage/integration': patch +--- + +A new Gerrit helper function (`buildGerritGitilesArchiveUrlFromLocation`) has been added. This +constructs a Gitiles URL to download an archive. It is similar to the existing +`buildGerritGitilesArchiveUrl` but also support content referenced by a full commit `SHA`. + +**DEPRECATIONS**: The function `buildGerritGitilesArchiveUrl` is deprecated, use the +`buildGerritGitilesArchiveUrlFromLocation` function instead. + +**DEPRECATIONS**: The function `parseGerritGitilesUrl` is deprecated, use the +`parseGitilesUrlRef` function instead. diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts index 14ca4c018e..2fbb85b9b1 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts @@ -20,7 +20,7 @@ import { registerMswTestHooks, } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; -import { NotModifiedError, NotFoundError } from '@backstage/errors'; +import { NotModifiedError } from '@backstage/errors'; import { GerritIntegration, readGerritIntegrationConfig, @@ -36,22 +36,11 @@ import { GerritUrlReader } from './GerritUrlReader'; import getRawBody from 'raw-body'; const mockDir = createMockDirectory({ mockOsTmpDir: true }); -const env = process.env; -process.env = { ...env, DISABLE_GERRIT_GITILES_REQUIREMENT: '1' }; const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); -const cloneMock = jest.fn(() => Promise.resolve()); -jest.mock('./git', () => ({ - Git: { - fromAuth: () => ({ - clone: cloneMock, - }), - }, -})); - // Gerrit processor without a gitilesBaseUrl configured const gerritProcessor = new GerritUrlReader( new GerritIntegration( @@ -96,12 +85,10 @@ describe.skip('GerritUrlReader', () => { beforeEach(() => { mockDir.clear(); - process.env = { ...env, DISABLE_GERRIT_GITILES_REQUIREMENT: '1' }; }); afterAll(() => { jest.clearAllMocks(); - process.env = env; }); describe('reader factory', () => { @@ -186,7 +173,24 @@ describe.skip('GerritUrlReader', () => { const buffer = await result.buffer(); expect(buffer.toString()).toBe(responseBuffer.toString()); }); - + it('should be able to read file contents of a commit as buffer', async () => { + worker.use( + rest.get( + 'https://gerrit.com/projects/web%2Fproject/commits/f775f9119c313c7ffc890d7908a45997273434d5/files/LICENSE/content', + (_, res, ctx) => { + return res( + ctx.status(200), + ctx.body(responseBuffer.toString('base64')), + ); + }, + ), + ); + const result = await gerritProcessor.readUrl( + 'https://gerrit.com/web/project/+/f775f9119c313c7ffc890d7908a45997273434d5/LICENSE', + ); + const buffer = await result.buffer(); + expect(buffer.toString()).toBe(responseBuffer.toString()); + }); it('should be able to read file contents as stream', async () => { worker.use( rest.get( @@ -258,6 +262,7 @@ describe.skip('GerritUrlReader', () => { const treeUrlGitiles = 'https://gerrit.com/gitiles/app/web/+/refs/heads/master/'; const etag = '52432507a70b677b5674b019c9a46b2e9f29d0a1'; + const sha = 'f775f9119c313c7ffc890d7908a45997273434d5'; const mkdocsContent = 'a repo fetched using git clone'; const mdContent = 'doc'; const repoArchiveBuffer = fs.readFileSync( @@ -302,6 +307,19 @@ describe.skip('GerritUrlReader', () => { ctx.body(repoArchiveDocsBuffer), ), ), + rest.get( + `https://gerrit.com/gitiles/app/web/\\+archive/${sha}.tar.gz`, + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=web-refs/heads/master.tar.gz', + ), + ctx.body(repoArchiveBuffer), + ), + ), ); }); @@ -330,8 +348,6 @@ describe.skip('GerritUrlReader', () => { const mdFile = await files[1].content(); expect(mdFile.toString()).toBe('site_name: Test\n'); - - expect(cloneMock).not.toHaveBeenCalled(); }); it('throws NotModifiedError for matching etags.', async () => { @@ -346,15 +362,17 @@ describe.skip('GerritUrlReader', () => { ); }); - it('throws NotFoundError if branch info not found.', async () => { + it('throws ResponseError if branch info not found.', async () => { worker.use( rest.get(branchAPIUrl, (_, res, ctx) => { return res(ctx.status(404, 'Not found.')); }), ); - await expect(gerritProcessor.readTree(treeUrl)).rejects.toThrow( - NotFoundError, + await expect( + gerritProcessor.readTree(treeUrl), + ).rejects.toMatchInlineSnapshot( + `[ResponseError: Request failed with 404 Not found.]`, ); }); @@ -386,8 +404,20 @@ describe.skip('GerritUrlReader', () => { const mdFile = await files[0].content(); expect(mdFile.toString()).toBe('# Test\n'); + }); + it('throws NotModifiedError for a known commit.', async () => { + const shaTreeUrl = `https://gerrit.com/app/web/+/${sha}/`; - expect(cloneMock).not.toHaveBeenCalled(); + await expect( + gerritProcessor.readTree(shaTreeUrl, { etag: sha }), + ).rejects.toThrow(NotModifiedError); + }); + it('can fetch files for a specifc sha.', async () => { + const response = await gerritProcessorWithGitiles.readTree( + `https://gerrit.com/gitiles/app/web/+/${sha}/`, + ); + + expect(response.etag).toBe(sha); }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts index 2ef9116208..835624627b 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts @@ -28,14 +28,18 @@ import { Readable } from 'stream'; import { GerritIntegration, ScmIntegrations, - buildGerritGitilesArchiveUrl, + buildGerritGitilesArchiveUrlFromLocation, getGerritBranchApiUrl, getGerritFileContentsApiUrl, getGerritRequestOptions, - parseGerritGitilesUrl, parseGerritJsonResponse, + parseGitilesUrlRef, } from '@backstage/integration'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + NotFoundError, + NotModifiedError, + ResponseError, +} from '@backstage/errors'; import { ReadTreeResponseFactory, ReaderFactory } from './types'; /** @@ -135,34 +139,9 @@ export class GerritUrlReader implements UrlReaderService { url: string, options?: UrlReaderServiceReadTreeOptions, ): Promise { - const apiUrl = getGerritBranchApiUrl(this.integration.config, url); - let response: Response; - try { - response = await fetch(apiUrl, { - method: 'GET', - ...getGerritRequestOptions(this.integration.config), - }); - } catch (e) { - throw new Error(`Unable to read branch state ${url}, ${e}`); - } + const urlRevision = await this.getRevisionForUrl(url, options); - if (response.status === 404) { - throw new NotFoundError(`Not found: ${url}`); - } - - if (!response.ok) { - throw new Error( - `${url} could not be read as ${apiUrl}, ${response.status} ${response.statusText}`, - ); - } - const branchInfo = (await parseGerritJsonResponse(response as any)) as { - revision: string; - }; - if (options?.etag === branchInfo.revision) { - throw new NotModifiedError(); - } - - return this.readTreeFromGitiles(url, branchInfo.revision, options); + return this.readTreeFromGitiles(url, urlRevision, options); } async search(): Promise { @@ -179,16 +158,10 @@ export class GerritUrlReader implements UrlReaderService { revision: string, options?: UrlReaderServiceReadTreeOptions, ) { - const { branch, filePath, project } = parseGerritGitilesUrl( + const archiveUrl = buildGerritGitilesArchiveUrlFromLocation( this.integration.config, url, ); - const archiveUrl = buildGerritGitilesArchiveUrl( - this.integration.config, - project, - branch, - filePath, - ); const archiveResponse = await fetch(archiveUrl, { ...getGerritRequestOptions(this.integration.config), // TODO(freben): The signal cast is there because pre-3.x versions of @@ -217,4 +190,41 @@ export class GerritUrlReader implements UrlReaderService { stripFirstDirectory: false, }); } + + private async getRevisionForUrl( + url: string, + options?: UrlReaderServiceReadTreeOptions, + ): Promise { + const { ref, refType } = parseGitilesUrlRef(this.integration.config, url); + // The url points to a static revision. + if (refType === 'sha') { + if (options?.etag === ref) { + throw new NotModifiedError(); + } + return ref; + } + + const apiUrl = getGerritBranchApiUrl(this.integration.config, url); + let response: Response; + try { + response = await fetch(apiUrl, { + method: 'GET', + ...getGerritRequestOptions(this.integration.config), + }); + } catch (e) { + throw new Error(`Unable to read branch state ${url}, ${e}`); + } + + if (!response.ok) { + throw await ResponseError.fromResponse(response); + } + + const branchInfo = (await parseGerritJsonResponse(response as any)) as { + revision: string; + }; + if (options?.etag === branchInfo.revision) { + throw new NotModifiedError(); + } + return branchInfo.revision; + } } diff --git a/packages/integration/report.api.md b/packages/integration/report.api.md index 5129dbb7c2..6bb48b8185 100644 --- a/packages/integration/report.api.md +++ b/packages/integration/report.api.md @@ -299,7 +299,7 @@ export type BitbucketServerIntegrationConfig = { password?: string; }; -// @public +// @public @deprecated export function buildGerritGitilesArchiveUrl( config: GerritIntegrationConfig, project: string, @@ -307,6 +307,12 @@ export function buildGerritGitilesArchiveUrl( filePath: string, ): string; +// @public +export function buildGerritGitilesArchiveUrlFromLocation( + config: GerritIntegrationConfig, + url: string, +): string; + // @public export class DefaultAzureCredentialsManager implements AzureCredentialsManager { static fromIntegrations( @@ -802,7 +808,7 @@ export interface IntegrationsByType { harness: ScmIntegrationsGroup; } -// @public +// @public @deprecated export function parseGerritGitilesUrl( config: GerritIntegrationConfig, url: string, diff --git a/packages/integration/src/gerrit/core.test.ts b/packages/integration/src/gerrit/core.test.ts index c031612c40..5899196f65 100644 --- a/packages/integration/src/gerrit/core.test.ts +++ b/packages/integration/src/gerrit/core.test.ts @@ -21,6 +21,7 @@ import { registerMswTestHooks } from '../helpers'; import { GerritIntegrationConfig } from './config'; import { buildGerritGitilesArchiveUrl, + buildGerritGitilesArchiveUrlFromLocation, buildGerritGitilesUrl, getGerritBranchApiUrl, getGerritCloneRepoUrl, @@ -115,6 +116,94 @@ describe('gerrit core', () => { }); }); + describe('buildGerritGitilesArchiveUrlFromLocation', () => { + const config: GerritIntegrationConfig = { + host: 'gerrit.com', + baseUrl: 'https://gerrit.com', + gitilesBaseUrl: 'https://gerrit.com/gitiles', + }; + const configWithPath: GerritIntegrationConfig = { + host: 'gerrit.com', + baseUrl: 'https://gerrit.com/gerrit', + gitilesBaseUrl: 'https://gerrit.com/gerrit/plugins/gitiles', + }; + const configWithDedicatedGitiles: GerritIntegrationConfig = { + host: 'gerrit.com', + baseUrl: 'https://gerrit.com/gerrit', + gitilesBaseUrl: 'https://dedicated-gitiles-server.com/gerrit/gitiles', + }; + it('can create an archive url for a branch', () => { + expect( + buildGerritGitilesArchiveUrlFromLocation( + config, + 'https://gerrit.com/gitiles/repo/+/refs/heads/dev/', + ), + ).toEqual( + 'https://gerrit.com/gitiles/repo/+archive/refs/heads/dev.tar.gz', + ); + }); + it('can create an archive url for a sha', () => { + expect( + buildGerritGitilesArchiveUrlFromLocation( + config, + 'https://gerrit.com/gitiles/repo/+/2846e8dc327ae2f60249983b1c3b96f42f205bae/', + ), + ).toEqual( + 'https://gerrit.com/gitiles/repo/+archive/2846e8dc327ae2f60249983b1c3b96f42f205bae.tar.gz', + ); + }); + it('can create an archive url for a sha with a specific directory', () => { + expect( + buildGerritGitilesArchiveUrlFromLocation( + config, + 'https://gerrit.com/gitiles/repo/+/2846e8dc327ae2f60249983b1c3b96f42f205bae/docs', + ), + ).toEqual( + 'https://gerrit.com/gitiles/repo/+archive/2846e8dc327ae2f60249983b1c3b96f42f205bae/docs.tar.gz', + ); + }); + it('can create an archive url for a specific directory', () => { + expect( + buildGerritGitilesArchiveUrlFromLocation( + config, + 'https://gerrit.com/gitiles/repo/+/refs/heads/dev/docs/', + ), + ).toEqual( + 'https://gerrit.com/gitiles/repo/+archive/refs/heads/dev/docs.tar.gz', + ); + }); + it('can create an authenticated url when auth is enabled and an url-path is used', () => { + const authConfig = { + ...configWithPath, + username: 'username', + password: 'password', + }; + expect( + buildGerritGitilesArchiveUrlFromLocation( + authConfig, + 'https://gerrit.com/gerrit/plugins/gitiles/repo/+/refs/heads/dev/docs/', + ), + ).toEqual( + 'https://gerrit.com/gerrit/a/plugins/gitiles/repo/+archive/refs/heads/dev/docs.tar.gz', + ); + }); + it('Cannot build an authenticated url when a dedicated Gitiles server is used', () => { + const authConfig = { + ...configWithDedicatedGitiles, + username: 'username', + password: 'password', + }; + expect(() => + buildGerritGitilesArchiveUrlFromLocation( + authConfig, + 'https://gerrit.com/gitiles/repo/+/refs/heads/dev/', + ), + ).toThrow( + 'Since the baseUrl (Gerrit) is not part of the gitilesBaseUrl, an authentication URL could not be constructed.', + ); + }); + }); + describe('buildGerritGitilesUrl', () => { it('can create an url from arguments', () => { const config: GerritIntegrationConfig = { @@ -426,6 +515,39 @@ describe('gerrit core', () => { 'https://gerrit.com/a/projects/web%2Fproject/branches/master/files/README.md/content', ); }); + it('can create an authenticated url for a commit.', () => { + const authConfig: GerritIntegrationConfig = { + host: 'gerrit.com', + baseUrl: 'https://gerrit.com', + gitilesBaseUrl: 'https://gerrit.com', + username: 'u', + password: 'u', + }; + + const authFileContentUrl = getGerritFileContentsApiUrl( + authConfig, + 'https://gerrit.com/web/project/+/157f862803d45b9d269f0e390f88aece1ded51e8/README.md', + ); + expect(authFileContentUrl).toEqual( + 'https://gerrit.com/a/projects/web%2Fproject/commits/157f862803d45b9d269f0e390f88aece1ded51e8/files/README.md/content', + ); + }); + it('will throw for unsupported ref types (tag).', () => { + const authConfig: GerritIntegrationConfig = { + host: 'gerrit.com', + baseUrl: 'https://gerrit.com', + gitilesBaseUrl: 'https://gerrit.com', + username: 'u', + password: 'u', + }; + + expect(() => + getGerritFileContentsApiUrl( + authConfig, + 'https://gerrit.com/modules/events-broker/+/refs/tags/v3.5.6/src/main/java/com/gerritforge/gerrit/eventbroker/BrokerApi.java', + ), + ).toThrow(/gitiles ref type/); + }); }); describe('parseGerritJsonResponse', () => { diff --git a/packages/integration/src/gerrit/core.ts b/packages/integration/src/gerrit/core.ts index e50ce97bbb..d44865881d 100644 --- a/packages/integration/src/gerrit/core.ts +++ b/packages/integration/src/gerrit/core.ts @@ -41,8 +41,9 @@ const GERRIT_BODY_PREFIX = ")]}'"; * * @param url - An URL pointing to a file stored in git. * @public + * @deprecated `parseGerritGitilesUrl` is deprecated. Use + * {@link parseGitilesUrlRef} instead. */ - export function parseGerritGitilesUrl( config: GerritIntegrationConfig, url: string, @@ -215,6 +216,8 @@ export function buildGerritGitilesUrl( * @param branch - The branch we will target. * @param filePath - The absolute file path. * @public + * @deprecated `buildGerritGitilesArchiveUrl` is deprecated. Use + * {@link buildGerritGitilesArchiveUrlFromLocation} instead. */ export function buildGerritGitilesArchiveUrl( config: GerritIntegrationConfig, @@ -229,6 +232,38 @@ export function buildGerritGitilesArchiveUrl( )}/${project}/+archive/refs/heads/${branch}${archiveName}`; } +/** + * Build a Gerrit Gitiles archive url from a Gitiles url. + * + * @param config - A Gerrit provider config. + * @param url - The gitiles url + * @public + */ +export function buildGerritGitilesArchiveUrlFromLocation( + config: GerritIntegrationConfig, + url: string, +): string { + const { + path: filePath, + ref, + project, + refType, + } = parseGitilesUrlRef(config, url); + const archiveName = + filePath === '/' || filePath === '' ? '.tar.gz' : `/${filePath}.tar.gz`; + if (refType === 'branch') { + return `${getGitilesAuthenticationUrl( + config, + )}/${project}/+archive/refs/heads/${ref}${archiveName}`; + } + if (refType === 'sha') { + return `${getGitilesAuthenticationUrl( + config, + )}/${project}/+archive/${ref}${archiveName}`; + } + throw new Error(`Unsupported gitiles ref type: ${refType}`); +} + /** * Return the authentication prefix. * @@ -324,13 +359,25 @@ export function getGerritFileContentsApiUrl( config: GerritIntegrationConfig, url: string, ) { - const { branch, filePath, project } = parseGerritGitilesUrl(config, url); + const { ref, refType, path, project } = parseGitilesUrlRef(config, url); - return `${config.baseUrl}${getAuthenticationPrefix( - config, - )}projects/${encodeURIComponent( - project, - )}/branches/${branch}/files/${encodeURIComponent(filePath)}/content`; + // https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-content + if (refType === 'branch') { + return `${config.baseUrl}${getAuthenticationPrefix( + config, + )}projects/${encodeURIComponent( + project, + )}/branches/${ref}/files/${encodeURIComponent(path)}/content`; + } + // https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-content-from-commit + if (refType === 'sha') { + return `${config.baseUrl}${getAuthenticationPrefix( + config, + )}projects/${encodeURIComponent( + project, + )}/commits/${ref}/files/${encodeURIComponent(path)}/content`; + } + throw new Error(`Unsupported gitiles ref type: ${refType}`); } /** diff --git a/packages/integration/src/gerrit/index.ts b/packages/integration/src/gerrit/index.ts index f2e5934c95..3df52bf02b 100644 --- a/packages/integration/src/gerrit/index.ts +++ b/packages/integration/src/gerrit/index.ts @@ -20,6 +20,7 @@ export { } from './config'; export { buildGerritGitilesArchiveUrl, + buildGerritGitilesArchiveUrlFromLocation, getGerritBranchApiUrl, getGerritCloneRepoUrl, getGerritFileContentsApiUrl,