Merge pull request #3673 from mateiidavid/implement-readtree-on-gitlab

Implement readTree for GitlabReader (#3547)
2020-12-17 11:10:55 +01:00
parent eae3d383d1 d8e9b75dc7
commit f671440858
5 changed files with 283 additions and 100 deletions
@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': patch
+'@backstage/integration': patch
+---
+
+Introduce readTree method for GitLab URL Reader
@@ -14,12 +14,14 @@
 * limitations under the License.
 */

+import { ConfigReader } from '@backstage/config';
+import { msw } from '@backstage/test-utils';
+import fs from 'fs';
 import { rest } from 'msw';
 import { setupServer } from 'msw/node';
-import { ConfigReader } from '@backstage/config';
+import path from 'path';
 import { getVoidLogger } from '../logging';
 import { GitlabUrlReader } from './GitlabUrlReader';
-import { msw } from '@backstage/test-utils';
 import { ReadTreeResponseFactory } from './tree';

 const logger = getVoidLogger();
@@ -30,105 +32,207 @@ const treeResponseFactory = ReadTreeResponseFactory.create({

 describe('GitlabUrlReader', () => {
  const worker = setupServer();
-
  msw.setupDefaultHandlers(worker);

-  beforeEach(() => {
-    worker.use(
-      rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
-        res(ctx.status(200), ctx.json({ id: 12345 })),
-      ),
-      rest.get('*', (req, res, ctx) =>
-        res(
-          ctx.status(200),
-          ctx.json({
-            url: req.url.toString(),
-            headers: req.headers.getAllHeaders(),
-          }),
+  describe('implementation', () => {
+    beforeEach(() => {
+      worker.use(
+        rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
+          res(ctx.status(200), ctx.json({ id: 12345 })),
        ),
-      ),
-    );
-  });
-
-  const createConfig = (token?: string) =>
-    new ConfigReader(
-      {
-        integrations: { gitlab: [{ host: 'gitlab.com', token }] },
-      },
-      'test-config',
-    );
-
-  it.each([
-    // Project URLs
-    {
-      url:
-        'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
-        headers: expect.objectContaining({
-          'private-token': '',
-        }),
-      }),
-    },
-    {
-      url:
-        'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
-      config: createConfig('0123456789'),
-      response: expect.objectContaining({
-        url:
-          'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
-        headers: expect.objectContaining({
-          'private-token': '0123456789',
-        }),
-      }),
-    },
-    {
-      url:
-        'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
-      }),
-    },
-
-    // Raw URLs
-    {
-      url: 'https://gitlab.example.com/a/b/blob/master/c.yaml',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url: 'https://gitlab.example.com/a/b/raw/master/c.yaml',
-      }),
-    },
-  ])('should handle happy path %#', async ({ url, config, response }) => {
-    const [{ reader }] = GitlabUrlReader.factory({
-      config,
-      logger,
-      treeResponseFactory,
+        rest.get('*', (req, res, ctx) =>
+          res(
+            ctx.status(200),
+            ctx.json({
+              url: req.url.toString(),
+              headers: req.headers.getAllHeaders(),
+            }),
+          ),
+        ),
+      );
    });

-    const data = await reader.read(url);
-    const res = await JSON.parse(data.toString('utf-8'));
-    expect(res).toEqual(response);
-  });
+    const createConfig = (token?: string) =>
+      new ConfigReader(
+        {
+          integrations: { gitlab: [{ host: 'gitlab.com', token }] },
+        },
+        'test-config',
+      );

-  it.each([
-    {
-      url: '',
-      config: createConfig(''),
-      error:
-        "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string",
-    },
-  ])('should handle error path %#', async ({ url, config, error }) => {
-    await expect(async () => {
+    it.each([
+      // Project URLs
+      {
+        url:
+          'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+          headers: expect.objectContaining({
+            'private-token': '',
+          }),
+        }),
+      },
+      {
+        url:
+          'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+        config: createConfig('0123456789'),
+        response: expect.objectContaining({
+          url:
+            'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+          headers: expect.objectContaining({
+            'private-token': '0123456789',
+          }),
+        }),
+      },
+      {
+        url:
+          'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+        }),
+      },
+
+      // Raw URLs
+      {
+        url: 'https://gitlab.example.com/a/b/blob/master/c.yaml',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url: 'https://gitlab.example.com/a/b/raw/master/c.yaml',
+        }),
+      },
+    ])('should handle happy path %#', async ({ url, config, response }) => {
      const [{ reader }] = GitlabUrlReader.factory({
        config,
        logger,
        treeResponseFactory,
      });
-      await reader.read(url);
-    }).rejects.toThrow(error);
+
+      const data = await reader.read(url);
+      const res = await JSON.parse(data.toString('utf-8'));
+      expect(res).toEqual(response);
+    });
+
+    it.each([
+      {
+        url: '',
+        config: createConfig(''),
+        error:
+          "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string",
+      },
+    ])('should handle error path %#', async ({ url, config, error }) => {
+      await expect(async () => {
+        const [{ reader }] = GitlabUrlReader.factory({
+          config,
+          logger,
+          treeResponseFactory,
+        });
+        await reader.read(url);
+      }).rejects.toThrow(error);
+    });
+  });
+
+  describe('readTree', () => {
+    const repoBuffer = fs.readFileSync(
+      path.resolve('src', 'reading', '__fixtures__', 'repo.zip'),
+    );
+
+    beforeEach(() => {
+      worker.use(
+        rest.get(
+          'https://gitlab.com/backstage/mock/-/archive/repo/mock-repo.zip',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+    });
+
+    it('returns the wanted files from an archive', async () => {
+      const processor = new GitlabUrlReader(
+        { host: 'gitlab.com' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://gitlab.com/backstage/mock/tree/repo',
+      );
+
+      const files = await response.files();
+      expect(files.length).toBe(2);
+
+      const indexMarkdownFile = await files[0].content();
+      const mkDocsFile = await files[1].content();
+
+      expect(mkDocsFile.toString()).toBe('site_name: Test\n');
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('returns the wanted files from hosted gitlab', async () => {
+      worker.use(
+        rest.get(
+          'https://git.mycompany.com/backstage/mock/-/archive/repo/mock-repo.zip',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+
+      const processor = new GitlabUrlReader(
+        { host: 'git.mycompany.com' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://git.mycompany.com/backstage/mock/tree/repo/docs',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(1);
+      const indexMarkdownFile = await files[0].content();
+
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('throws an error when branch is not specified', async () => {
+      const processor = new GitlabUrlReader(
+        { host: 'gitlab.com' },
+        { treeResponseFactory },
+      );
+
+      await expect(
+        processor.readTree('https://gitlab.com/backstage/mock'),
+      ).rejects.toThrow(
+        'GitLab URL must contain a branch to be able to fetch its tree',
+      );
+    });
+
+    it('returns the wanted files from an archive with a subpath', async () => {
+      const processor = new GitlabUrlReader(
+        { host: 'gitlab.com' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://gitlab.com/backstage/mock/tree/repo/docs',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(1);
+      const indexMarkdownFile = await files[0].content();
+
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
  });
 });
@@ -21,22 +21,37 @@ import {
  readGitLabIntegrationConfigs,
 } from '@backstage/integration';
 import fetch from 'cross-fetch';
-import { NotFoundError } from '../errors';
-import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
+import { InputError, NotFoundError } from '../errors';
+import { ReadTreeResponseFactory } from './tree';
+import {
+  ReaderFactory,
+  ReadTreeOptions,
+  ReadTreeResponse,
+  UrlReader,
+} from './types';
+import parseGitUri from 'git-url-parse';
+import { Readable } from 'stream';

 export class GitlabUrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  private readonly treeResponseFactory: ReadTreeResponseFactory;
+
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
    const configs = readGitLabIntegrationConfigs(
      config.getOptionalConfigArray('integrations.gitlab') ?? [],
    );
    return configs.map(options => {
-      const reader = new GitlabUrlReader(options);
+      const reader = new GitlabUrlReader(options, { treeResponseFactory });
      const predicate = (url: URL) => url.host === options.host;
      return { reader, predicate };
    });
  };

-  constructor(private readonly options: GitLabIntegrationConfig) {}
+  constructor(
+    private readonly options: GitLabIntegrationConfig,
+    deps: { treeResponseFactory: ReadTreeResponseFactory },
+  ) {
+    this.treeResponseFactory = deps.treeResponseFactory;
+  }

  async read(url: string): Promise<Buffer> {
    const builtUrl = await getGitLabFileFetchUrl(url, this.options);
@@ -59,8 +74,45 @@ export class GitlabUrlReader implements UrlReader {
    throw new Error(message);
  }

-  readTree(): Promise<ReadTreeResponse> {
-    throw new Error('GitlabUrlReader does not implement readTree');
+  async readTree(
+    url: string,
+    options?: ReadTreeOptions,
+  ): Promise<ReadTreeResponse> {
+    const {
+      name: repoName,
+      ref,
+      protocol,
+      resource,
+      full_name,
+      filepath,
+    } = parseGitUri(url);
+
+    if (!ref) {
+      throw new InputError(
+        'GitLab URL must contain a branch to be able to fetch its tree',
+      );
+    }
+
+    const archive = `${protocol}://${resource}/${full_name}/-/archive/${ref}/${repoName}-${ref}.zip`;
+    const response = await fetch(
+      archive,
+      getGitLabRequestOptions(this.options),
+    );
+    if (!response.ok) {
+      const msg = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
+      if (response.status === 404) {
+        throw new NotFoundError(msg);
+      }
+      throw new Error(msg);
+    }
+
+    const path = filepath ? `${repoName}-${ref}/${filepath}/` : '';
+
+    return this.treeResponseFactory.fromZipArchive({
+      stream: (response.body as unknown) as Readable,
+      path,
+      filter: options?.filter,
+    });
  }

  toString() {
@@ -41,7 +41,10 @@ describe('readGitLabIntegrationConfig', () => {

  it('inserts the defaults if missing', () => {
    const output = readGitLabIntegrationConfig(buildConfig({}));
-    expect(output).toEqual({ host: 'gitlab.com' });
+    expect(output).toEqual({
+      host: 'gitlab.com',
+      apiBaseUrl: 'gitlab.com/api/v4',
+    });
  });

  it('rejects funky configs', () => {
@@ -18,6 +18,7 @@ import { Config } from '@backstage/config';
 import { isValidHost } from '../helpers';

 const GITLAB_HOST = 'gitlab.com';
+const GITLAB_API_BASE_URL = 'gitlab.com/api/v4';

 /**
 * The configuration parameters for a single GitLab integration.
@@ -28,6 +29,17 @@ export type GitLabIntegrationConfig = {
   */
  host: string;

+  /**
+   * The base URL of the API of this provider, e.g. "https://gitlab.com/api/v4",
+   * with no trailing slash.
+   *
+   * May be omitted specifically for GitLab; then it will be deduced.
+   *
+   * The API will always be preferred if both its base URL and a token are
+   * present.
+   */
+  apiBaseUrl?: string;
+
  /**
   * The authorization token to use for requests this provider.
   *
@@ -45,6 +57,7 @@ export function readGitLabIntegrationConfig(
  config: Config,
 ): GitLabIntegrationConfig {
  const host = config.getOptionalString('host') ?? GITLAB_HOST;
+  let apiBaseUrl = config.getOptionalString('apiBaseUrl');
  const token = config.getOptionalString('token');

  if (!isValidHost(host)) {
@@ -53,7 +66,12 @@ export function readGitLabIntegrationConfig(
    );
  }

-  return { host, token };
+  if (apiBaseUrl) {
+    apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
+  } else if (host === GITLAB_HOST) {
+    apiBaseUrl = GITLAB_API_BASE_URL;
+  }
+  return { host, token, apiBaseUrl };
 }

 /**