Implement readTree for GitlabReader (#3547)

* This change adds an implementation for the readTree method in GitlabReader. The implementation is similar to the readTree method in GithubReader with the exception of the 'path' variable used for subpaths in the archive. To facilitate this, GitlabIntegrationConfig now has an apiUrl field which was missing for Gitlab but was present for Bitbucket & GH. * As part of this change, there are also new tests added to GitlabReader to mirror what has been done with the GH reader. For now the archive format chosen is 'zip', follow-up action includes having a look at whether tar.gz is preferable. Signed-off-by: Matei David <madavid@expediagroup.com>
2020-12-10 12:34:09 +00:00
parent d62be2f822
commit 47930a36f7
3 changed files with 273 additions and 99 deletions
@@ -14,12 +14,14 @@
 * limitations under the License.
 */

+import { ConfigReader } from '@backstage/config';
+import { msw } from '@backstage/test-utils';
+import fs from 'fs';
 import { rest } from 'msw';
 import { setupServer } from 'msw/node';
-import { ConfigReader } from '@backstage/config';
+import path from 'path';
 import { getVoidLogger } from '../logging';
 import { GitlabUrlReader } from './GitlabUrlReader';
-import { msw } from '@backstage/test-utils';
 import { ReadTreeResponseFactory } from './tree';

 const logger = getVoidLogger();
@@ -30,105 +32,207 @@ const treeResponseFactory = ReadTreeResponseFactory.create({

 describe('GitlabUrlReader', () => {
  const worker = setupServer();
-
  msw.setupDefaultHandlers(worker);

-  beforeEach(() => {
-    worker.use(
-      rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
-        res(ctx.status(200), ctx.json({ id: 12345 })),
-      ),
-      rest.get('*', (req, res, ctx) =>
-        res(
-          ctx.status(200),
-          ctx.json({
-            url: req.url.toString(),
-            headers: req.headers.getAllHeaders(),
-          }),
+  describe('implementation', () => {
+    beforeEach(() => {
+      worker.use(
+        rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
+          res(ctx.status(200), ctx.json({ id: 12345 })),
        ),
-      ),
-    );
-  });
-
-  const createConfig = (token?: string) =>
-    new ConfigReader(
-      {
-        integrations: { gitlab: [{ host: 'gitlab.com', token }] },
-      },
-      'test-config',
-    );
-
-  it.each([
-    // Project URLs
-    {
-      url:
-        'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
-        headers: expect.objectContaining({
-          'private-token': '',
-        }),
-      }),
-    },
-    {
-      url:
-        'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
-      config: createConfig('0123456789'),
-      response: expect.objectContaining({
-        url:
-          'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
-        headers: expect.objectContaining({
-          'private-token': '0123456789',
-        }),
-      }),
-    },
-    {
-      url:
-        'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
-      }),
-    },
-
-    // Raw URLs
-    {
-      url: 'https://gitlab.example.com/a/b/blob/master/c.yaml',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url: 'https://gitlab.example.com/a/b/raw/master/c.yaml',
-      }),
-    },
-  ])('should handle happy path %#', async ({ url, config, response }) => {
-    const [{ reader }] = GitlabUrlReader.factory({
-      config,
-      logger,
-      treeResponseFactory,
+        rest.get('*', (req, res, ctx) =>
+          res(
+            ctx.status(200),
+            ctx.json({
+              url: req.url.toString(),
+              headers: req.headers.getAllHeaders(),
+            }),
+          ),
+        ),
+      );
    });

-    const data = await reader.read(url);
-    const res = await JSON.parse(data.toString('utf-8'));
-    expect(res).toEqual(response);
-  });
+    const createConfig = (token?: string) =>
+      new ConfigReader(
+        {
+          integrations: { gitlab: [{ host: 'gitlab.com', token }] },
+        },
+        'test-config',
+      );

-  it.each([
-    {
-      url: '',
-      config: createConfig(''),
-      error:
-        "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string",
-    },
-  ])('should handle error path %#', async ({ url, config, error }) => {
-    await expect(async () => {
+    it.each([
+      // Project URLs
+      {
+        url:
+          'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+          headers: expect.objectContaining({
+            'private-token': '',
+          }),
+        }),
+      },
+      {
+        url:
+          'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+        config: createConfig('0123456789'),
+        response: expect.objectContaining({
+          url:
+            'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+          headers: expect.objectContaining({
+            'private-token': '0123456789',
+          }),
+        }),
+      },
+      {
+        url:
+          'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+        }),
+      },
+
+      // Raw URLs
+      {
+        url: 'https://gitlab.example.com/a/b/blob/master/c.yaml',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url: 'https://gitlab.example.com/a/b/raw/master/c.yaml',
+        }),
+      },
+    ])('should handle happy path %#', async ({ url, config, response }) => {
      const [{ reader }] = GitlabUrlReader.factory({
        config,
        logger,
        treeResponseFactory,
      });
-      await reader.read(url);
-    }).rejects.toThrow(error);
+
+      const data = await reader.read(url);
+      const res = await JSON.parse(data.toString('utf-8'));
+      expect(res).toEqual(response);
+    });
+
+    it.each([
+      {
+        url: '',
+        config: createConfig(''),
+        error:
+          "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string",
+      },
+    ])('should handle error path %#', async ({ url, config, error }) => {
+      await expect(async () => {
+        const [{ reader }] = GitlabUrlReader.factory({
+          config,
+          logger,
+          treeResponseFactory,
+        });
+        await reader.read(url);
+      }).rejects.toThrow(error);
+    });
+  });
+
+  describe('readTree', () => {
+    const repoBuffer = fs.readFileSync(
+      path.resolve('src', 'reading', '__fixtures__', 'repo.zip'),
+    );
+
+    beforeEach(() => {
+      worker.use(
+        rest.get(
+          'https://gitlab.com/backstage/mock/-/archive/repo/mock-repo.zip',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+    });
+
+    it('returns the wanted files from an archive', async () => {
+      const processor = new GitlabUrlReader(
+        { host: 'gitlab.com' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://gitlab.com/backstage/mock/tree/repo',
+      );
+
+      const files = await response.files();
+      expect(files.length).toBe(2);
+
+      const indexMarkdownFile = await files[0].content();
+      const mkDocsFile = await files[1].content();
+
+      expect(mkDocsFile.toString()).toBe('site_name: Test\n');
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('returns the wanted files from hosted gitlab', async () => {
+      worker.use(
+        rest.get(
+          'https://git.mycompany.com/backstage/mock/-/archive/repo/mock-repo.zip',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+
+      const processor = new GitlabUrlReader(
+        { host: 'git.mycompany.com' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://git.mycompany.com/backstage/mock/tree/repo/docs',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(1);
+      const indexMarkdownFile = await files[0].content();
+
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('throws an error when branch is not specified', async () => {
+      const processor = new GitlabUrlReader(
+        { host: 'gitlab.com' },
+        { treeResponseFactory },
+      );
+
+      await expect(
+        processor.readTree('https://gitlab.com/backstage/mock'),
+      ).rejects.toThrow(
+        'GitLab URL must contain a branch to be able to fetch its tree',
+      );
+    });
+
+    it('returns the wanted files from an archive with a subpath', async () => {
+      const processor = new GitlabUrlReader(
+        { host: 'gitlab.com' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://gitlab.com/backstage/mock/tree/repo/docs',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(1);
+      const indexMarkdownFile = await files[0].content();
+
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
  });
 });
@@ -21,22 +21,37 @@ import {
  readGitLabIntegrationConfigs,
 } from '@backstage/integration';
 import fetch from 'cross-fetch';
-import { NotFoundError } from '../errors';
-import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
+import { InputError, NotFoundError } from '../errors';
+import { ReadTreeResponseFactory } from './tree';
+import {
+  ReaderFactory,
+  ReadTreeOptions,
+  ReadTreeResponse,
+  UrlReader,
+} from './types';
+import parseGitUri from 'git-url-parse';
+import { Readable } from 'stream';

 export class GitlabUrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  private readonly treeResponseFactory: ReadTreeResponseFactory;
+
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
    const configs = readGitLabIntegrationConfigs(
      config.getOptionalConfigArray('integrations.gitlab') ?? [],
    );
    return configs.map(options => {
-      const reader = new GitlabUrlReader(options);
+      const reader = new GitlabUrlReader(options, { treeResponseFactory });
      const predicate = (url: URL) => url.host === options.host;
      return { reader, predicate };
    });
  };

-  constructor(private readonly options: GitLabIntegrationConfig) {}
+  constructor(
+    private readonly options: GitLabIntegrationConfig,
+    deps: { treeResponseFactory: ReadTreeResponseFactory },
+  ) {
+    this.treeResponseFactory = deps.treeResponseFactory;
+  }

  async read(url: string): Promise<Buffer> {
    const builtUrl = await getGitLabFileFetchUrl(url, this.options);
@@ -59,8 +74,45 @@ export class GitlabUrlReader implements UrlReader {
    throw new Error(message);
  }

-  readTree(): Promise<ReadTreeResponse> {
-    throw new Error('GitlabUrlReader does not implement readTree');
+  async readTree(
+    url: string,
+    options?: ReadTreeOptions,
+  ): Promise<ReadTreeResponse> {
+    const {
+      name: repoName,
+      ref,
+      protocol,
+      resource,
+      full_name,
+      filepath,
+    } = parseGitUri(url);
+
+    if (!ref) {
+      throw new InputError(
+        'GitLab URL must contain a branch to be able to fetch its tree',
+      );
+    }
+
+    const archive = `${protocol}://${resource}/${full_name}/-/archive/${ref}/${repoName}-${ref}.zip`;
+    const response = await fetch(
+      archive,
+      getGitLabRequestOptions(this.options),
+    );
+    if (!response.ok) {
+      const msg = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
+      if (response.status === 404) {
+        throw new NotFoundError(msg);
+      }
+      throw new Error(msg);
+    }
+
+    const path = filepath ? `${repoName}-${ref}/${filepath}/` : '';
+
+    return this.treeResponseFactory.fromZipArchive({
+      stream: (response.body as unknown) as Readable,
+      path,
+      filter: options?.filter,
+    });
  }

  toString() {
@@ -18,6 +18,7 @@ import { Config } from '@backstage/config';
 import { isValidHost } from '../helpers';

 const GITLAB_HOST = 'gitlab.com';
+const GITLAB_API_BASE_URL = 'gitlab.com/api/v4';

 /**
 * The configuration parameters for a single GitLab integration.
@@ -28,6 +29,17 @@ export type GitLabIntegrationConfig = {
   */
  host: string;

+  /**
+   * The base URL of the API of this provider, e.g. "https://gitlab.com/api/v4",
+   * with no trailing slash.
+   *
+   * May be omitted specifically for GitLab; then it will be deduced.
+   *
+   * The API will always be preferred if both its base URL and a token are
+   * present.
+   */
+  apiBaseUrl?: string;
+
  /**
   * The authorization token to use for requests this provider.
   *
@@ -45,6 +57,7 @@ export function readGitLabIntegrationConfig(
  config: Config,
 ): GitLabIntegrationConfig {
  const host = config.getOptionalString('host') ?? GITLAB_HOST;
+  let apiBaseUrl = config.getOptionalString('apiBaseUrl');
  const token = config.getOptionalString('token');

  if (!isValidHost(host)) {
@@ -53,7 +66,12 @@ export function readGitLabIntegrationConfig(
    );
  }

-  return { host, token };
+  if (apiBaseUrl) {
+    apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
+  } else if (host === GITLAB_HOST) {
+    apiBaseUrl = GITLAB_API_BASE_URL;
+  }
+  return { host, token, apiBaseUrl };
 }

 /**