Implement readTree of Gitea provider to support Techdocs

Signed-off-by: Daniel Mecsei <mecseid@gmail.com>
2023-09-28 09:33:07 +02:00
parent 998d5a704e
commit 870db76a45
7 changed files with 336 additions and 28 deletions
@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': minor
+'@backstage/integration': minor
+---
+
+Implemented readTree for Gitea provider to basically support Techdocs functionality
@@ -68,6 +68,7 @@ See [TechDocs Architecture](architecture.md) to get an overview of where the bel
 | Gerrit                       | Yes ✅         |
 | GitLab                       | Yes ✅         |
 | GitLab Enterprise            | Yes ✅         |
+| Gitea                        | Yes ✅         |

 ### File storage providers

@@ -26,6 +26,8 @@ import { DefaultReadTreeResponseFactory } from './tree';
 import getRawBody from 'raw-body';
 import { GiteaUrlReader } from './GiteaUrlReader';
 import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import fs from 'fs-extra';
+import path from 'path';

 const treeResponseFactory = DefaultReadTreeResponseFactory.create({
  config: new ConfigReader({}),
@@ -47,6 +49,7 @@ const giteaProcessor = new GiteaUrlReader(
      }),
    ),
  ),
+  { treeResponseFactory },
 );

 const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
@@ -247,4 +250,82 @@ describe('GiteaUrlReader', () => {
      ).rejects.toThrow(NotModifiedError);
    });
  });
+
+  describe('readTree', () => {
+    const commitHash = '3bdd5457286abdf920db4b77bf2fef79a06190c2';
+
+    const repoBuffer = fs.readFileSync(
+      path.resolve(__dirname, '__fixtures__/mock-main.tar.gz'),
+    );
+
+    beforeEach(() => {
+      worker.use(
+        rest.get(
+          'https://gitea.com/api/v1/repos/owner/project/git/commits/branch2',
+          (_, res, ctx) => {
+            return res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/json'),
+              ctx.json({ sha: commitHash }),
+            );
+          },
+        ),
+      );
+    });
+
+    it('should be able to get archive', async () => {
+      worker.use(
+        rest.get(
+          'https://gitea.com/api/v1/repos/owner/project/archive/branch2.tar.gz',
+          (_, res, ctx) => {
+            return res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/gzip'),
+              ctx.set(
+                'content-disposition',
+                'attachment; filename=backstage-mock.tar.gz',
+              ),
+              ctx.body(repoBuffer),
+            );
+          },
+        ),
+      );
+
+      const response = await giteaProcessor.readTree(
+        'https://gitea.com/owner/project/src/branch/branch2',
+      );
+      expect(response.etag).toBe(commitHash);
+
+      const files = await response.files();
+      expect(files.length).toBe(2);
+    });
+
+    it('should return not modified', async () => {
+      await expect(
+        giteaProcessor.readTree(
+          'https://gitea.com/owner/project/src/branch/branch2',
+          {
+            etag: commitHash,
+          },
+        ),
+      ).rejects.toThrow(NotModifiedError);
+    });
+
+    it('should return not found', async () => {
+      worker.use(
+        rest.get(
+          'https://gitea.com/api/v1/repos/owner/project/git/commits/branch3',
+          (_, res, ctx) => {
+            return res(ctx.status(404));
+          },
+        ),
+      );
+
+      await expect(
+        giteaProcessor.readTree(
+          'https://gitea.com/owner/project/src/branch/branch3',
+        ),
+      ).rejects.toThrow(NotFoundError);
+    });
+  });
 });
@@ -14,15 +14,21 @@
 * limitations under the License.
 */
 import {
-  getGiteaRequestOptions,
  getGiteaFileContentsUrl,
+  getGiteaArchiveUrl,
+  getGiteaLatestCommitUrl,
+  parseGiteaUrl,
+  getGiteaRequestOptions,
  GiteaIntegration,
  ScmIntegrations,
 } from '@backstage/integration';
-import { ReadUrlOptions, ReadUrlResponse } from './types';
 import {
  ReaderFactory,
+  ReadTreeOptions,
  ReadTreeResponse,
+  ReadTreeResponseFactory,
+  ReadUrlOptions,
+  ReadUrlResponse,
  SearchResponse,
  UrlReader,
 } from './types';
@@ -42,11 +48,11 @@ import { parseLastModified } from './util';
 * @public
 */
 export class GiteaUrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
    return ScmIntegrations.fromConfig(config)
      .gitea.list()
      .map(integration => {
-        const reader = new GiteaUrlReader(integration);
+        const reader = new GiteaUrlReader(integration, { treeResponseFactory });
        const predicate = (url: URL) => {
          return url.host === integration.config.host;
        };
@@ -54,7 +60,12 @@ export class GiteaUrlReader implements UrlReader {
      });
  };

-  constructor(private readonly integration: GiteaIntegration) {}
+  constructor(
+    private readonly integration: GiteaIntegration,
+    private readonly deps: {
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
+  ) {}

  async read(url: string): Promise<Buffer> {
    const response = await this.readUrl(url);
@@ -113,9 +124,38 @@ export class GiteaUrlReader implements UrlReader {
    throw new Error(message);
  }

-  readTree(): Promise<ReadTreeResponse> {
-    throw new Error('GiteaUrlReader readTree not implemented.');
+  async readTree(
+    url: string,
+    options?: ReadTreeOptions,
+  ): Promise<ReadTreeResponse> {
+    const lastCommitHash = await this.getLastCommitHash(url);
+    if (options?.etag && options.etag === lastCommitHash) {
+      throw new NotModifiedError();
+    }
+
+    const archiveUri = getGiteaArchiveUrl(this.integration.config, url);
+
+    let response: Response;
+    try {
+      response = await fetch(archiveUri, {
+        method: 'GET',
+        ...getGiteaRequestOptions(this.integration.config),
+        signal: options?.signal as any,
+      });
+    } catch (e) {
+      throw new Error(`Unable to read ${archiveUri}, ${e}`);
+    }
+
+    const parsedUri = parseGiteaUrl(this.integration.config, url);
+
+    return this.deps.treeResponseFactory.fromTarArchive({
+      stream: Readable.from(response.body),
+      subpath: parsedUri.path,
+      etag: lastCommitHash,
+      filter: options?.filter,
+    });
  }
+
  search(): Promise<SearchResponse> {
    throw new Error('GiteaUrlReader search not implemented.');
  }
@@ -126,4 +166,22 @@ export class GiteaUrlReader implements UrlReader {
      this.integration.config.password,
    )}}`;
  }
+
+  private async getLastCommitHash(url: string): Promise<string> {
+    const commitUri = getGiteaLatestCommitUrl(this.integration.config, url);
+
+    const response = await fetch(
+      commitUri,
+      getGiteaRequestOptions(this.integration.config),
+    );
+    if (!response.ok) {
+      const message = `Failed to retrieve latest commit information from ${commitUri}, ${response.status} ${response.statusText}`;
+      if (response.status === 404) {
+        throw new NotFoundError(message);
+      }
+      throw new Error(message);
+    }
+
+    return (await response.json()).sha;
+  }
 }
@@ -18,9 +18,12 @@ import { setupServer } from 'msw/node';
 import { setupRequestMockHandlers } from '../helpers';
 import { GiteaIntegrationConfig } from './config';
 import {
+  getGiteaArchiveUrl,
  getGiteaEditContentsUrl,
  getGiteaFileContentsUrl,
+  getGiteaLatestCommitUrl,
  getGiteaRequestOptions,
+  parseGiteaUrl,
 } from './core';

 describe('gitea core', () => {
@@ -59,6 +62,38 @@ describe('gitea core', () => {
    });
  });

+  describe('getGiteaArchiveUrl', () => {
+    it('can create an url from arguments', () => {
+      const config: GiteaIntegrationConfig = {
+        host: 'gitea.example.com',
+      };
+      expect(
+        getGiteaArchiveUrl(
+          config,
+          'https://gitea.example.com/owner/repo/src/branch/branch_name',
+        ),
+      ).toEqual(
+        'https://gitea.example.com/api/v1/repos/owner/repo/archive/branch_name.tar.gz',
+      );
+    });
+  });
+
+  describe('getGiteaLatestCommitUrl', () => {
+    it('can create an url from arguments', () => {
+      const config: GiteaIntegrationConfig = {
+        host: 'gitea.example.com',
+      };
+      expect(
+        getGiteaLatestCommitUrl(
+          config,
+          'https://gitea.example.com/owner/repo/src/branch/branch_name/',
+        ),
+      ).toEqual(
+        'https://gitea.example.com/api/v1/repos/owner/repo/git/commits/branch_name',
+      );
+    });
+  });
+
  describe('getGerritRequestOptions', () => {
    it('adds token header when only a password is specified', () => {
      const authRequest: GiteaIntegrationConfig = {
@@ -90,4 +125,61 @@ describe('gitea core', () => {
      ).toEqual(basicAuthentication);
    });
  });
+
+  describe('parseGiteaUrl', () => {
+    it('can fetch gitea url', () => {
+      const config: GiteaIntegrationConfig = {
+        host: 'gitea.example.com',
+      };
+      expect(
+        parseGiteaUrl(
+          config,
+          'https://gitea.example.com/owner/repo/src/branch/branch_name/',
+        ),
+      ).toEqual({
+        url: 'https://gitea.example.com',
+        owner: 'owner',
+        name: 'repo',
+        ref: 'branch_name',
+        path: '',
+      });
+    });
+
+    it('provide path without starting slash', () => {
+      const config: GiteaIntegrationConfig = {
+        host: 'gitea.example.com',
+      };
+      expect(
+        parseGiteaUrl(
+          config,
+          'https://gitea.example.com/owner/repo/src/branch/branch_name/simple/path',
+        ),
+      ).toEqual({
+        url: 'https://gitea.example.com',
+        owner: 'owner',
+        name: 'repo',
+        ref: 'branch_name',
+        path: 'simple/path',
+      });
+    });
+
+    it('use base url if provided', () => {
+      const config: GiteaIntegrationConfig = {
+        host: 'gitea.example.com',
+        baseUrl: 'https://base-gitea.example.com',
+      };
+      expect(
+        parseGiteaUrl(
+          config,
+          'https://base-gitea.example.com/owner/repo/src/branch/branch_name/',
+        ),
+      ).toEqual({
+        url: 'https://base-gitea.example.com',
+        owner: 'owner',
+        name: 'repo',
+        ref: 'branch_name',
+        path: '',
+      });
+    });
+  });
 });
@@ -33,16 +33,8 @@ export function getGiteaEditContentsUrl(
  config: GiteaIntegrationConfig,
  url: string,
 ) {
-  try {
-    const baseUrl = config.baseUrl ?? `https://${config.host}`;
-    const [_blank, owner, name, _src, _branch, ref, ...path] = url
-      .replace(baseUrl, '')
-      .split('/');
-    const pathWithoutSlash = path.join('/').replace(/^\//, '');
-    return `${baseUrl}/${owner}/${name}/_edit/${ref}/${pathWithoutSlash}`;
-  } catch (e) {
-    throw new Error(`Incorrect URL: ${url}, ${e}`);
-  }
+  const giteaUrl = parseGiteaUrl(config, url);
+  return `${giteaUrl.url}/${giteaUrl.owner}/${giteaUrl.name}/_edit/${giteaUrl.ref}/${giteaUrl.path}`;
 }

 /**
@@ -63,17 +55,52 @@ export function getGiteaFileContentsUrl(
  config: GiteaIntegrationConfig,
  url: string,
 ) {
-  try {
-    const baseUrl = config.baseUrl ?? `https://${config.host}`;
-    const [_blank, owner, name, _src, _branch, ref, ...path] = url
-      .replace(baseUrl, '')
-      .split('/');
-    const pathWithoutSlash = path.join('/').replace(/^\//, '');
+  const giteaUrl = parseGiteaUrl(config, url);
+  return `${giteaUrl.url}/api/v1/repos/${giteaUrl.owner}/${giteaUrl.name}/contents/${giteaUrl.path}?ref=${giteaUrl.ref}`;
+}

-    return `${baseUrl}/api/v1/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`;
-  } catch (e) {
-    throw new Error(`Incorrect URL: ${url}, ${e}`);
-  }
+/**
+ * Given a URL pointing to a repository/path, returns a URL
+ * for archive contents of the repository.
+ *
+ * @remarks
+ *
+ * Converts
+ * from: https://gitea.com/a/b/src/branchname
+ * or:   https://gitea.com/api/v1/repos/a/b/archive/branchname.tar.gz
+ *
+ * @param url - A URL pointing to a repository/path
+ * @param config - The relevant provider config
+ * @public
+ */
+export function getGiteaArchiveUrl(
+  config: GiteaIntegrationConfig,
+  url: string,
+) {
+  const giteaUrl = parseGiteaUrl(config, url);
+  return `${giteaUrl.url}/api/v1/repos/${giteaUrl.owner}/${giteaUrl.name}/archive/${giteaUrl.ref}.tar.gz`;
+}
+
+/**
+ * Given a URL pointing to a repository branch, returns a URL
+ * for latest commit information.
+ *
+ * @remarks
+ *
+ * Converts
+ * from: https://gitea.com/a/b/src/branchname
+ * or:   https://gitea.com/api/v1/repos/a/b/git/commits/branchname
+ *
+ * @param url - A URL pointing to a repository branch
+ * @param config - The relevant provider config
+ * @public
+ */
+export function getGiteaLatestCommitUrl(
+  config: GiteaIntegrationConfig,
+  url: string,
+) {
+  const giteaUrl = parseGiteaUrl(config, url);
+  return `${giteaUrl.url}/api/v1/repos/${giteaUrl.owner}/${giteaUrl.name}/git/commits/${giteaUrl.ref}`;
 }

 /**
@@ -104,3 +131,39 @@ export function getGiteaRequestOptions(config: GiteaIntegrationConfig): {
    headers,
  };
 }
+
+/**
+ * Return parsed git url properties.
+ *
+ * @param config - A Gitea provider config
+ * @param url - A URL pointing to a repository
+ * @public
+ */
+export function parseGiteaUrl(
+  config: GiteaIntegrationConfig,
+  url: string,
+): {
+  url: string;
+  owner: string;
+  name: string;
+  ref: string;
+  path: string;
+} {
+  const baseUrl = config.baseUrl ?? `https://${config.host}`;
+  try {
+    const [_blank, owner, name, _src, _branch, ref, ...path] = url
+      .replace(baseUrl, '')
+      .split('/');
+    const pathWithoutSlash = path.join('/').replace(/^\//, '');
+
+    return {
+      url: baseUrl,
+      owner: owner,
+      name: name,
+      ref: ref,
+      path: pathWithoutSlash,
+    };
+  } catch (e) {
+    throw new Error(`Incorrect URL: ${url}, ${e}`);
+  }
+}
@@ -14,6 +14,13 @@
 * limitations under the License.
 */
 export { GiteaIntegration } from './GiteaIntegration';
-export { getGiteaRequestOptions, getGiteaFileContentsUrl } from './core';
+export {
+  getGiteaEditContentsUrl,
+  getGiteaFileContentsUrl,
+  getGiteaArchiveUrl,
+  getGiteaLatestCommitUrl,
+  getGiteaRequestOptions,
+  parseGiteaUrl,
+} from './core';
 export { readGiteaConfig } from './config';
 export type { GiteaIntegrationConfig } from './config';