diff --git a/.changeset/sixty-houses-agree.md b/.changeset/sixty-houses-agree.md new file mode 100644 index 0000000000..4a4ec0744d --- /dev/null +++ b/.changeset/sixty-houses-agree.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': minor +'@backstage/integration': minor +--- + +Implemented readTree for Gitea provider to basically support Techdocs functionality diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index 3251451cce..c6e1fd42ad 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -68,6 +68,7 @@ See [TechDocs Architecture](architecture.md) to get an overview of where the bel | Gerrit | Yes ✅ | | GitLab | Yes ✅ | | GitLab Enterprise | Yes ✅ | +| Gitea | Yes ✅ | ### File storage providers diff --git a/packages/backend-common/src/reading/GiteaUrlReader.test.ts b/packages/backend-common/src/reading/GiteaUrlReader.test.ts index 1860d9f461..a941d2e5aa 100644 --- a/packages/backend-common/src/reading/GiteaUrlReader.test.ts +++ b/packages/backend-common/src/reading/GiteaUrlReader.test.ts @@ -26,6 +26,8 @@ import { DefaultReadTreeResponseFactory } from './tree'; import getRawBody from 'raw-body'; import { GiteaUrlReader } from './GiteaUrlReader'; import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import fs from 'fs-extra'; +import path from 'path'; const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), @@ -47,6 +49,7 @@ const giteaProcessor = new GiteaUrlReader( }), ), ), + { treeResponseFactory }, ); const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { @@ -247,4 +250,82 @@ describe('GiteaUrlReader', () => { ).rejects.toThrow(NotModifiedError); }); }); + + describe('readTree', () => { + const commitHash = '3bdd5457286abdf920db4b77bf2fef79a06190c2'; + + const repoBuffer = fs.readFileSync( + path.resolve(__dirname, '__fixtures__/mock-main.tar.gz'), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://gitea.com/api/v1/repos/owner/project/git/commits/branch2', + (_, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ sha: commitHash }), + ); + }, + ), + ); + }); + + it('should be able to get archive', async () => { + worker.use( + rest.get( + 'https://gitea.com/api/v1/repos/owner/project/archive/branch2.tar.gz', + (_, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock.tar.gz', + ), + ctx.body(repoBuffer), + ); + }, + ), + ); + + const response = await giteaProcessor.readTree( + 'https://gitea.com/owner/project/src/branch/branch2', + ); + expect(response.etag).toBe(commitHash); + + const files = await response.files(); + expect(files.length).toBe(2); + }); + + it('should return not modified', async () => { + await expect( + giteaProcessor.readTree( + 'https://gitea.com/owner/project/src/branch/branch2', + { + etag: commitHash, + }, + ), + ).rejects.toThrow(NotModifiedError); + }); + + it('should return not found', async () => { + worker.use( + rest.get( + 'https://gitea.com/api/v1/repos/owner/project/git/commits/branch3', + (_, res, ctx) => { + return res(ctx.status(404)); + }, + ), + ); + + await expect( + giteaProcessor.readTree( + 'https://gitea.com/owner/project/src/branch/branch3', + ), + ).rejects.toThrow(NotFoundError); + }); + }); }); diff --git a/packages/backend-common/src/reading/GiteaUrlReader.ts b/packages/backend-common/src/reading/GiteaUrlReader.ts index 34542e878e..c8fab45157 100644 --- a/packages/backend-common/src/reading/GiteaUrlReader.ts +++ b/packages/backend-common/src/reading/GiteaUrlReader.ts @@ -14,15 +14,21 @@ * limitations under the License. */ import { - getGiteaRequestOptions, getGiteaFileContentsUrl, + getGiteaArchiveUrl, + getGiteaLatestCommitUrl, + parseGiteaUrl, + getGiteaRequestOptions, GiteaIntegration, ScmIntegrations, } from '@backstage/integration'; -import { ReadUrlOptions, ReadUrlResponse } from './types'; import { ReaderFactory, + ReadTreeOptions, ReadTreeResponse, + ReadTreeResponseFactory, + ReadUrlOptions, + ReadUrlResponse, SearchResponse, UrlReader, } from './types'; @@ -42,11 +48,11 @@ import { parseLastModified } from './util'; * @public */ export class GiteaUrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { return ScmIntegrations.fromConfig(config) .gitea.list() .map(integration => { - const reader = new GiteaUrlReader(integration); + const reader = new GiteaUrlReader(integration, { treeResponseFactory }); const predicate = (url: URL) => { return url.host === integration.config.host; }; @@ -54,7 +60,12 @@ export class GiteaUrlReader implements UrlReader { }); }; - constructor(private readonly integration: GiteaIntegration) {} + constructor( + private readonly integration: GiteaIntegration, + private readonly deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ) {} async read(url: string): Promise { const response = await this.readUrl(url); @@ -113,9 +124,38 @@ export class GiteaUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { - throw new Error('GiteaUrlReader readTree not implemented.'); + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const lastCommitHash = await this.getLastCommitHash(url); + if (options?.etag && options.etag === lastCommitHash) { + throw new NotModifiedError(); + } + + const archiveUri = getGiteaArchiveUrl(this.integration.config, url); + + let response: Response; + try { + response = await fetch(archiveUri, { + method: 'GET', + ...getGiteaRequestOptions(this.integration.config), + signal: options?.signal as any, + }); + } catch (e) { + throw new Error(`Unable to read ${archiveUri}, ${e}`); + } + + const parsedUri = parseGiteaUrl(this.integration.config, url); + + return this.deps.treeResponseFactory.fromTarArchive({ + stream: Readable.from(response.body), + subpath: parsedUri.path, + etag: lastCommitHash, + filter: options?.filter, + }); } + search(): Promise { throw new Error('GiteaUrlReader search not implemented.'); } @@ -126,4 +166,22 @@ export class GiteaUrlReader implements UrlReader { this.integration.config.password, )}}`; } + + private async getLastCommitHash(url: string): Promise { + const commitUri = getGiteaLatestCommitUrl(this.integration.config, url); + + const response = await fetch( + commitUri, + getGiteaRequestOptions(this.integration.config), + ); + if (!response.ok) { + const message = `Failed to retrieve latest commit information from ${commitUri}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + return (await response.json()).sha; + } } diff --git a/packages/integration/src/gitea/core.test.ts b/packages/integration/src/gitea/core.test.ts index 125f3077ef..fe6381d52e 100644 --- a/packages/integration/src/gitea/core.test.ts +++ b/packages/integration/src/gitea/core.test.ts @@ -18,9 +18,12 @@ import { setupServer } from 'msw/node'; import { setupRequestMockHandlers } from '../helpers'; import { GiteaIntegrationConfig } from './config'; import { + getGiteaArchiveUrl, getGiteaEditContentsUrl, getGiteaFileContentsUrl, + getGiteaLatestCommitUrl, getGiteaRequestOptions, + parseGiteaUrl, } from './core'; describe('gitea core', () => { @@ -59,6 +62,38 @@ describe('gitea core', () => { }); }); + describe('getGiteaArchiveUrl', () => { + it('can create an url from arguments', () => { + const config: GiteaIntegrationConfig = { + host: 'gitea.example.com', + }; + expect( + getGiteaArchiveUrl( + config, + 'https://gitea.example.com/owner/repo/src/branch/branch_name', + ), + ).toEqual( + 'https://gitea.example.com/api/v1/repos/owner/repo/archive/branch_name.tar.gz', + ); + }); + }); + + describe('getGiteaLatestCommitUrl', () => { + it('can create an url from arguments', () => { + const config: GiteaIntegrationConfig = { + host: 'gitea.example.com', + }; + expect( + getGiteaLatestCommitUrl( + config, + 'https://gitea.example.com/owner/repo/src/branch/branch_name/', + ), + ).toEqual( + 'https://gitea.example.com/api/v1/repos/owner/repo/git/commits/branch_name', + ); + }); + }); + describe('getGerritRequestOptions', () => { it('adds token header when only a password is specified', () => { const authRequest: GiteaIntegrationConfig = { @@ -90,4 +125,61 @@ describe('gitea core', () => { ).toEqual(basicAuthentication); }); }); + + describe('parseGiteaUrl', () => { + it('can fetch gitea url', () => { + const config: GiteaIntegrationConfig = { + host: 'gitea.example.com', + }; + expect( + parseGiteaUrl( + config, + 'https://gitea.example.com/owner/repo/src/branch/branch_name/', + ), + ).toEqual({ + url: 'https://gitea.example.com', + owner: 'owner', + name: 'repo', + ref: 'branch_name', + path: '', + }); + }); + + it('provide path without starting slash', () => { + const config: GiteaIntegrationConfig = { + host: 'gitea.example.com', + }; + expect( + parseGiteaUrl( + config, + 'https://gitea.example.com/owner/repo/src/branch/branch_name/simple/path', + ), + ).toEqual({ + url: 'https://gitea.example.com', + owner: 'owner', + name: 'repo', + ref: 'branch_name', + path: 'simple/path', + }); + }); + + it('use base url if provided', () => { + const config: GiteaIntegrationConfig = { + host: 'gitea.example.com', + baseUrl: 'https://base-gitea.example.com', + }; + expect( + parseGiteaUrl( + config, + 'https://base-gitea.example.com/owner/repo/src/branch/branch_name/', + ), + ).toEqual({ + url: 'https://base-gitea.example.com', + owner: 'owner', + name: 'repo', + ref: 'branch_name', + path: '', + }); + }); + }); }); diff --git a/packages/integration/src/gitea/core.ts b/packages/integration/src/gitea/core.ts index ee3fec12fc..95caa24c06 100644 --- a/packages/integration/src/gitea/core.ts +++ b/packages/integration/src/gitea/core.ts @@ -33,16 +33,8 @@ export function getGiteaEditContentsUrl( config: GiteaIntegrationConfig, url: string, ) { - try { - const baseUrl = config.baseUrl ?? `https://${config.host}`; - const [_blank, owner, name, _src, _branch, ref, ...path] = url - .replace(baseUrl, '') - .split('/'); - const pathWithoutSlash = path.join('/').replace(/^\//, ''); - return `${baseUrl}/${owner}/${name}/_edit/${ref}/${pathWithoutSlash}`; - } catch (e) { - throw new Error(`Incorrect URL: ${url}, ${e}`); - } + const giteaUrl = parseGiteaUrl(config, url); + return `${giteaUrl.url}/${giteaUrl.owner}/${giteaUrl.name}/_edit/${giteaUrl.ref}/${giteaUrl.path}`; } /** @@ -63,17 +55,52 @@ export function getGiteaFileContentsUrl( config: GiteaIntegrationConfig, url: string, ) { - try { - const baseUrl = config.baseUrl ?? `https://${config.host}`; - const [_blank, owner, name, _src, _branch, ref, ...path] = url - .replace(baseUrl, '') - .split('/'); - const pathWithoutSlash = path.join('/').replace(/^\//, ''); + const giteaUrl = parseGiteaUrl(config, url); + return `${giteaUrl.url}/api/v1/repos/${giteaUrl.owner}/${giteaUrl.name}/contents/${giteaUrl.path}?ref=${giteaUrl.ref}`; +} - return `${baseUrl}/api/v1/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`; - } catch (e) { - throw new Error(`Incorrect URL: ${url}, ${e}`); - } +/** + * Given a URL pointing to a repository/path, returns a URL + * for archive contents of the repository. + * + * @remarks + * + * Converts + * from: https://gitea.com/a/b/src/branchname + * or: https://gitea.com/api/v1/repos/a/b/archive/branchname.tar.gz + * + * @param url - A URL pointing to a repository/path + * @param config - The relevant provider config + * @public + */ +export function getGiteaArchiveUrl( + config: GiteaIntegrationConfig, + url: string, +) { + const giteaUrl = parseGiteaUrl(config, url); + return `${giteaUrl.url}/api/v1/repos/${giteaUrl.owner}/${giteaUrl.name}/archive/${giteaUrl.ref}.tar.gz`; +} + +/** + * Given a URL pointing to a repository branch, returns a URL + * for latest commit information. + * + * @remarks + * + * Converts + * from: https://gitea.com/a/b/src/branchname + * or: https://gitea.com/api/v1/repos/a/b/git/commits/branchname + * + * @param url - A URL pointing to a repository branch + * @param config - The relevant provider config + * @public + */ +export function getGiteaLatestCommitUrl( + config: GiteaIntegrationConfig, + url: string, +) { + const giteaUrl = parseGiteaUrl(config, url); + return `${giteaUrl.url}/api/v1/repos/${giteaUrl.owner}/${giteaUrl.name}/git/commits/${giteaUrl.ref}`; } /** @@ -104,3 +131,39 @@ export function getGiteaRequestOptions(config: GiteaIntegrationConfig): { headers, }; } + +/** + * Return parsed git url properties. + * + * @param config - A Gitea provider config + * @param url - A URL pointing to a repository + * @public + */ +export function parseGiteaUrl( + config: GiteaIntegrationConfig, + url: string, +): { + url: string; + owner: string; + name: string; + ref: string; + path: string; +} { + const baseUrl = config.baseUrl ?? `https://${config.host}`; + try { + const [_blank, owner, name, _src, _branch, ref, ...path] = url + .replace(baseUrl, '') + .split('/'); + const pathWithoutSlash = path.join('/').replace(/^\//, ''); + + return { + url: baseUrl, + owner: owner, + name: name, + ref: ref, + path: pathWithoutSlash, + }; + } catch (e) { + throw new Error(`Incorrect URL: ${url}, ${e}`); + } +} diff --git a/packages/integration/src/gitea/index.ts b/packages/integration/src/gitea/index.ts index 6b951b1fb0..071509d699 100644 --- a/packages/integration/src/gitea/index.ts +++ b/packages/integration/src/gitea/index.ts @@ -14,6 +14,13 @@ * limitations under the License. */ export { GiteaIntegration } from './GiteaIntegration'; -export { getGiteaRequestOptions, getGiteaFileContentsUrl } from './core'; +export { + getGiteaEditContentsUrl, + getGiteaFileContentsUrl, + getGiteaArchiveUrl, + getGiteaLatestCommitUrl, + getGiteaRequestOptions, + parseGiteaUrl, +} from './core'; export { readGiteaConfig } from './config'; export type { GiteaIntegrationConfig } from './config';