Expose ZipArchiveResponse through ReadTreeResponseFactory

2020-11-16 16:08:37 +01:00
parent 9367dde00d
commit 34a51a288a
4 changed files with 221 additions and 99 deletions
@@ -14,11 +14,13 @@
 * limitations under the License.
 */

+import fs from 'fs';
+import path from 'path';
 import { rest } from 'msw';
 import { setupServer } from 'msw/node';
 import { ConfigReader } from '@backstage/config';
 import { getVoidLogger } from '../logging';
-import { AzureUrlReader } from './AzureUrlReader';
+import { AzureUrlReader, getDownloadUrl } from './AzureUrlReader';
 import { msw } from '@backstage/test-utils';
 import { ReadTreeResponseFactory } from './tree';

@@ -32,104 +34,165 @@ describe('AzureUrlReader', () => {
  const worker = setupServer();
  msw.setupDefaultHandlers(worker);

-  beforeEach(() => {
-    worker.use(
-      rest.get('*', (req, res, ctx) =>
-        res(
-          ctx.status(200),
-          ctx.json({
-            url: req.url.toString(),
-            headers: req.headers.getAllHeaders(),
-          }),
+  describe('read', () => {
+    beforeEach(() => {
+      worker.use(
+        rest.get('*', (req, res, ctx) =>
+          res(
+            ctx.status(200),
+            ctx.json({
+              url: req.url.toString(),
+              headers: req.headers.getAllHeaders(),
+            }),
+          ),
        ),
-      ),
-    );
-  });
-
-  const createConfig = (token?: string) =>
-    new ConfigReader(
-      {
-        integrations: { azure: [{ host: 'dev.azure.com', token }] },
-      },
-      'test-config',
-    );
-
-  it.each([
-    {
-      url:
-        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master',
-      }),
-    },
-    {
-      url:
-        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml',
-      }),
-    },
-    {
-      url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
-      config: createConfig('0123456789'),
-      response: expect.objectContaining({
-        headers: expect.objectContaining({
-          authorization: 'Basic OjAxMjM0NTY3ODk=',
-        }),
-      }),
-    },
-    {
-      url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
-      config: createConfig(undefined),
-      response: expect.objectContaining({
-        headers: expect.not.objectContaining({
-          authorization: expect.anything(),
-        }),
-      }),
-    },
-  ])('should handle happy path %#', async ({ url, config, response }) => {
-    const [{ reader }] = AzureUrlReader.factory({
-      config,
-      logger,
-      treeResponseFactory,
+      );
    });

-    const data = await reader.read(url);
-    const res = await JSON.parse(data.toString('utf-8'));
-    expect(res).toEqual(response);
-  });
+    const createConfig = (token?: string) =>
+      new ConfigReader(
+        {
+          integrations: { azure: [{ host: 'dev.azure.com', token }] },
+        },
+        'test-config',
+      );

-  it.each([
-    {
-      url: 'https://api.com/a/b/blob/master/path/to/c.yaml',
-      config: createConfig(),
-      error:
-        'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
-    },
-    {
-      url: 'com/a/b/blob/master/path/to/c.yaml',
-      config: createConfig(),
-      error:
-        'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
-    },
-    {
-      url: '',
-      config: createConfig(''),
-      error:
-        "Invalid type in config for key 'integrations.azure[0].token' in 'test-config', got empty-string, wanted string",
-    },
-  ])('should handle error path %#', async ({ url, config, error }) => {
-    await expect(async () => {
+    it.each([
+      {
+        url:
+          'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master',
+        }),
+      },
+      {
+        url:
+          'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml',
+        }),
+      },
+      {
+        url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
+        config: createConfig('0123456789'),
+        response: expect.objectContaining({
+          headers: expect.objectContaining({
+            authorization: 'Basic OjAxMjM0NTY3ODk=',
+          }),
+        }),
+      },
+      {
+        url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
+        config: createConfig(undefined),
+        response: expect.objectContaining({
+          headers: expect.not.objectContaining({
+            authorization: expect.anything(),
+          }),
+        }),
+      },
+    ])('should handle happy path %#', async ({ url, config, response }) => {
      const [{ reader }] = AzureUrlReader.factory({
        config,
        logger,
        treeResponseFactory,
      });
-      await reader.read(url);
-    }).rejects.toThrow(error);
+
+      const data = await reader.read(url);
+      const res = await JSON.parse(data.toString('utf-8'));
+      expect(res).toEqual(response);
+    });
+
+    it.each([
+      {
+        url: 'https://api.com/a/b/blob/master/path/to/c.yaml',
+        config: createConfig(),
+        error:
+          'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
+      },
+      {
+        url: 'com/a/b/blob/master/path/to/c.yaml',
+        config: createConfig(),
+        error:
+          'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
+      },
+      {
+        url: '',
+        config: createConfig(''),
+        error:
+          "Invalid type in config for key 'integrations.azure[0].token' in 'test-config', got empty-string, wanted string",
+      },
+    ])('should handle error path %#', async ({ url, config, error }) => {
+      await expect(async () => {
+        const [{ reader }] = AzureUrlReader.factory({
+          config,
+          logger,
+          treeResponseFactory,
+        });
+        await reader.read(url);
+      }).rejects.toThrow(error);
+    });
+  });
+
+  describe('readTree', () => {
+    const repoBuffer = fs.readFileSync(
+      path.resolve('src', 'reading', '__fixtures__', 'repo.zip'),
+    );
+
+    beforeEach(() => {
+      worker.use(
+        rest.get(
+          'https://dev.azure.com/organization/project/_apis/git/repositories/repository/items',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+    });
+
+    it('returns the wanted files from an archive', async () => {
+      const processor = new AzureUrlReader(
+        {
+          host: 'dev.azure.com',
+        },
+        treeResponseFactory,
+      );
+
+      const response = await processor.readTree(
+        'https://dev.azure.com/organization/project/_git/repository',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(2);
+      const mkDocsFile = await files[1].content();
+      const indexMarkdownFile = await files[0].content();
+
+      expect(mkDocsFile.toString()).toBe('site_name: Test\n');
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+  });
+
+  describe('getDownloadUrl', () => {
+    it('add no scopePath if no path is specified', async () => {
+      const result = getDownloadUrl(
+        'https://dev.azure.com/organization/project/_git/repository',
+      );
+
+      expect(result.searchParams.get('scopePath')).toBeNull();
+    });
+
+    it('add the scopePath if a path is specified', async () => {
+      const result = getDownloadUrl(
+        'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs',
+      );
+      expect(result.searchParams.get('scopePath')).toEqual('docs');
+    });
  });
 });
@@ -19,22 +19,53 @@ import {
  readAzureIntegrationConfigs,
 } from '@backstage/integration';
 import fetch from 'cross-fetch';
+import { Readable } from 'stream';
+import parseGitUri from 'git-url-parse';
 import { NotFoundError } from '../errors';
-import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
+import {
+  ReaderFactory,
+  ReadTreeOptions,
+  ReadTreeResponse,
+  UrlReader,
+} from './types';
+import { ReadTreeResponseFactory } from './tree';
+
+export function getDownloadUrl(url: string): URL {
+  const {
+    name: repoName,
+    owner: project,
+    organization,
+    protocol,
+    resource,
+    filepath,
+  } = parseGitUri(url);
+
+  // scopePath will limit the downloaded content
+  // /docs will only download the docs folder and everything below it
+  // /docs/index.md will only download index.md but put it in the root of the archive
+  const scopePath = filepath ? `&scopePath=${filepath}` : '';
+
+  return new URL(
+    `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`,
+  );
+}

 export class AzureUrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
    const configs = readAzureIntegrationConfigs(
      config.getOptionalConfigArray('integrations.azure') ?? [],
    );
    return configs.map(options => {
-      const reader = new AzureUrlReader(options);
+      const reader = new AzureUrlReader(options, treeResponseFactory);
      const predicate = (url: URL) => url.host === options.host;
      return { reader, predicate };
    });
  };

-  constructor(private readonly options: AzureIntegrationConfig) {
+  constructor(
+    private readonly options: AzureIntegrationConfig,
+    private readonly treeResponseFactory: ReadTreeResponseFactory,
+  ) {
    if (options.host !== 'dev.azure.com') {
      throw Error(
        `Azure integration currently only supports 'dev.azure.com', tried to use host '${options.host}'`,
@@ -64,8 +95,26 @@ export class AzureUrlReader implements UrlReader {
    throw new Error(message);
  }

-  readTree(): Promise<ReadTreeResponse> {
-    throw new Error('AzureUrlReader does not implement readTree');
+  async readTree(
+    url: string,
+    options?: ReadTreeOptions,
+  ): Promise<ReadTreeResponse> {
+    const response = await fetch(getDownloadUrl(url).toString(), {
+      ...this.getRequestOptions(),
+      headers: { Accept: 'application/zip' },
+    });
+    if (!response.ok) {
+      const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
+      if (response.status === 404) {
+        throw new NotFoundError(message);
+      }
+      throw new Error(message);
+    }
+
+    return this.treeResponseFactory.fromZipArchive({
+      stream: (response.body as unknown) as Readable,
+      filter: options?.filter,
+    });
  }

  // Converts
@@ -207,7 +207,7 @@ export class GithubUrlReader implements UrlReader {

    const path = `${repoName}-${ref}/${filepath}`;

-    return this.deps.treeResponseFactory.fromArchive({
+    return this.deps.treeResponseFactory.fromTarArchive({
      // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want
      //               to stick to using that in exclusively backend code.
      stream: (response.body as unknown) as Readable,
@@ -19,6 +19,7 @@ import { Readable } from 'stream';
 import { Config } from '@backstage/config';
 import { ReadTreeResponse } from '../types';
 import { TarArchiveResponse } from './TarArchiveResponse';
+import { ZipArchiveResponse } from './ZipArchiveResponse';

 type FromArchiveOptions = {
  // A binary stream of a tar archive.
@@ -39,7 +40,7 @@ export class ReadTreeResponseFactory {

  constructor(private readonly workDir: string) {}

-  async fromArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
+  async fromTarArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
    return new TarArchiveResponse(
      options.stream,
      options.path ?? '',
@@ -47,4 +48,13 @@ export class ReadTreeResponseFactory {
      options.filter,
    );
  }
+
+  async fromZipArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
+    return new ZipArchiveResponse(
+      options.stream,
+      options.path ?? '',
+      this.workDir,
+      options.filter,
+    );
+  }
 }