readTree method on UrlReader (#2925)

* Basic implementation of readTree on github and use it in techdocs * Added a UrlPreparer and cleaned up some minor things * Download entire archive and filter out the wanted files in readTree * Added test for dir output of tree reader * Fixed formatting * Fixed formatting * Some cleanups and fixes * Fix typing issues * Fix prettier * Fix prettier * Removed unused dependency * Fixed comments on PR * Prettier * Moved @types/fs-extra to devDeps * Fixed another PR comment * Added a test for getDocFilesFromRepository * Prettier
2020-11-09 16:28:51 +01:00
parent 9e336a57a5
commit f059eaf755
15 changed files with 450 additions and 8 deletions
@@ -36,11 +36,13 @@
    "@types/cors": "^2.8.6",
    "@types/express": "^4.17.6",
    "compression": "^1.7.4",
+    "concat-stream": "^2.0.0",
    "cors": "^2.8.5",
    "cross-fetch": "^3.0.6",
    "express": "^4.17.1",
    "express-prom-bundle": "^6.1.0",
    "express-promise-router": "^3.0.3",
+    "fs-extra": "^9.0.1",
    "git-url-parse": "^11.4.0",
    "helmet": "^4.0.0",
    "knex": "^0.21.6",
@@ -51,6 +53,7 @@
    "prom-client": "^12.0.0",
    "selfsigned": "^1.10.7",
    "stoppable": "^1.1.0",
+    "tar": "^6.0.5",
    "winston": "^3.2.1"
  },
  "peerDependencies": {
@@ -64,17 +67,24 @@
  "devDependencies": {
    "@backstage/cli": "^0.2.0",
    "@types/compression": "^1.7.0",
+    "@types/concat-stream": "^1.6.0",
+    "@types/fs-extra": "^9.0.3",
    "@types/http-errors": "^1.6.3",
    "@types/minimist": "^1.2.0",
+    "@types/mock-fs": "^4.13.0",
    "@types/morgan": "^1.9.0",
+    "@types/recursive-readdir": "^2.2.0",
    "@types/stoppable": "^1.1.0",
    "@types/supertest": "^2.0.8",
+    "@types/tar": "^4.0.3",
    "@types/webpack-env": "^1.15.2",
    "@types/yaml": "^1.9.7",
    "get-port": "^5.1.1",
    "http-errors": "^1.7.3",
    "jest": "^26.0.1",
+    "mock-fs": "^4.13.0",
    "msw": "^0.21.2",
+    "recursive-readdir": "^2.2.2",
    "supertest": "^4.0.2"
  },
  "files": [
@@ -15,6 +15,9 @@
 */

 import { ConfigReader } from '@backstage/config';
+import { setupServer } from 'msw/node';
+import { msw } from '@backstage/test-utils';
+import { rest } from 'msw';
 import {
  getApiRequestOptions,
  getApiUrl,
@@ -24,6 +27,10 @@ import {
  ProviderConfig,
  readConfig,
 } from './GithubUrlReader';
+import fs from 'fs';
+import path from 'path';
+import mockfs from 'mock-fs';
+import recursive from 'recursive-readdir';

 describe('GithubUrlReader', () => {
  describe('getApiRequestOptions', () => {
@@ -230,4 +237,77 @@ describe('GithubUrlReader', () => {
      );
    });
  });
+
+  describe('readTree', () => {
+    const worker = setupServer();
+
+    msw.setupDefaultHandlers(worker);
+
+    const repoBuffer = fs.readFileSync(
+      path.resolve('src', 'reading', '__fixtures__', 'repo.tar.gz'),
+    );
+
+    beforeEach(() => {
+      worker.use(
+        rest.get(
+          'https://github.com/spotify/mock/archive/repo.tar.gz',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/x-gzip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+    });
+
+    it('returns the wanted files from an archive', async () => {
+      const processor = new GithubUrlReader({
+        host: 'github.com',
+      });
+
+      const response = await processor.readTree(
+        'https://github.com/spotify/mock',
+        'repo',
+        ['mkdocs.yml', 'docs'],
+      );
+
+      const files = response.files();
+
+      const mkDocsFile = await files[0].content();
+      const indexMarkdownFile = await files[1].content();
+
+      expect(mkDocsFile.toString()).toBe('site_name: Test\n');
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('returns a folder path from an archive', async () => {
+      const processor = new GithubUrlReader({
+        host: 'github.com',
+      });
+
+      const response = await processor.readTree(
+        'https://github.com/spotify/mock',
+        'repo',
+        ['mkdocs.yml', 'docs'],
+      );
+
+      mockfs();
+      const directory = await response.dir('/tmp/fs');
+
+      const writtenToDirectory = fs.existsSync(directory);
+      const paths = await recursive(directory);
+      mockfs.restore();
+
+      expect(writtenToDirectory).toBe(true);
+      expect(paths.sort()).toEqual(
+        [
+          '/tmp/fs/mock-repo/docs/index.md',
+          '/tmp/fs/mock-repo/mkdocs.yml',
+        ].sort(),
+      );
+
+      worker.resetHandlers();
+    });
+  });
 });
@@ -18,7 +18,12 @@ import { Config } from '@backstage/config';
 import parseGitUri from 'git-url-parse';
 import fetch from 'cross-fetch';
 import { NotFoundError } from '../errors';
-import { ReaderFactory, UrlReader } from './types';
+import { ReaderFactory, ReadTreeResponse, UrlReader, File } from './types';
+import tar from 'tar';
+import fs from 'fs-extra';
+import concatStream from 'concat-stream';
+import path from 'path';
+import os from 'os';

 /**
 * The configuration parameters for a single GitHub API provider.
@@ -229,6 +234,92 @@ export class GithubUrlReader implements UrlReader {
    throw new Error(message);
  }

+  private async getRepositoryArchive(
+    repoUrl: string,
+    branchName: string,
+  ): Promise<Response> {
+    return fetch(new URL(`${repoUrl}/archive/${branchName}.tar.gz`).toString());
+  }
+
+  private async writeBufferToFile(
+    filePath: string,
+    content: Buffer,
+  ): Promise<void> {
+    await fs.outputFile(filePath, content.toString());
+  }
+
+  async readTree(
+    repoUrl: string,
+    branchName: string,
+    paths: Array<string>,
+  ): Promise<ReadTreeResponse> {
+    const { name: repoName } = parseGitUri(repoUrl);
+
+    const repoArchive = await this.getRepositoryArchive(repoUrl, branchName);
+
+    const files: File[] = [];
+    return new Promise(resolve => {
+      const parser = new (tar.Parse as any)({
+        filter: (path: string) =>
+          !!paths.filter(file => {
+            return path.startsWith(`${repoName}-${branchName}/${file}`);
+          }).length,
+        onentry: (entry: tar.ReadEntry) => {
+          if (entry.type === 'Directory') {
+            entry.resume();
+            return;
+          }
+
+          const contentPromise: Promise<Buffer> = new Promise(res => {
+            entry.pipe(concatStream(res));
+          });
+
+          files.push({
+            path: entry.path,
+            content: () => contentPromise,
+          });
+
+          entry.resume();
+        },
+      });
+
+      // @ts-ignore Typescript doesn't consider .pipe a method on ReadableStream. Don't know why.
+      repoArchive.body?.pipe(parser).on('finish', () => {
+        resolve({
+          files: () => {
+            return files;
+          },
+          archive: () => {
+            return new Promise(resolve =>
+              resolve(Buffer.from('Archive is not yet implemented')),
+            );
+          },
+          dir: (outDir: string | undefined) => {
+            const targetDirectory =
+              outDir || fs.mkdtempSync(path.join(os.tmpdir(), 'backstage-'));
+
+            return new Promise((res, rej) => {
+              Promise.all(
+                files.map(async file => {
+                  return this.writeBufferToFile(
+                    `${targetDirectory}/${file.path}`,
+                    await file.content(),
+                  );
+                }),
+              )
+                .then(() => {
+                  res(`${targetDirectory}/${repoName}-${branchName}`);
+                })
+                .catch(err => {
+                  rej(err);
+                });
+            });
+          },
+        });
+      });
+    });
+  }
+
  toString() {
    const { host, token } = this.config;
    return `github{host=${host},authed=${Boolean(token)}}`;
@@ -14,7 +14,7 @@
 * limitations under the License.
 */

-import { UrlReader, UrlReaderPredicateTuple } from './types';
+import { ReadTreeResponse, UrlReader, UrlReaderPredicateTuple } from './types';

 type Options = {
  // UrlReader to fall back to if no other reader is matched
@@ -53,6 +53,33 @@ export class UrlReaderPredicateMux implements UrlReader {
    throw new Error(`No reader found that could handle '${url}'`);
  }

+  readTree(
+    repoUrl: string,
+    branchName: string,
+    paths: Array<string>,
+  ): Promise<ReadTreeResponse> {
+    const parsed = new URL(repoUrl);
+
+    for (const { predicate, reader } of this.readers) {
+      if (predicate(parsed)) {
+        if (reader.readTree) return reader.readTree(repoUrl, branchName, paths);
+        throw new Error(
+          `Trying to call readTree on UrlReader which does not support the feature.`,
+        );
+      }
+    }
+
+    if (this.fallback) {
+      if (this.fallback.readTree)
+        return this.fallback.readTree(repoUrl, branchName, paths);
+      throw new Error(
+        `Trying to call readTree on UrlReader which does not support the feature.`,
+      );
+    }
+
+    throw new Error(`No reader found that could handle '${repoUrl}'`);
+  }
+
  toString() {
    return `predicateMux{readers=${this.readers
      .map(t => t.reader)
@@ -14,7 +14,7 @@
 * limitations under the License.
 */

-export type { UrlReader } from './types';
+export type { UrlReader, ReadTreeResponse } from './types';
 export { UrlReaders } from './UrlReaders';
 export { AzureUrlReader } from './AzureUrlReader';
 export { BitbucketUrlReader } from './BitbucketUrlReader';
@@ -22,6 +22,11 @@ import { Config } from '@backstage/config';
 */
 export type UrlReader = {
  read(url: string): Promise<Buffer>;
+  readTree?(
+    repoUrl: string,
+    branchName: string,
+    paths: Array<string>,
+  ): Promise<ReadTreeResponse>;
 };

 export type UrlReaderPredicateTuple = {
@@ -37,3 +42,14 @@ export type ReaderFactory = (options: {
  config: Config;
  logger: Logger;
 }) => UrlReaderPredicateTuple[];
+
+export type File = {
+  path: string;
+  content(): Promise<Buffer>;
+};
+
+export type ReadTreeResponse = {
+  files(): File[];
+  archive(): Promise<Buffer>;
+  dir(outDir?: string): Promise<string>;
+};