readTree method on UrlReader (#2925)

* Basic implementation of readTree on github and use it in techdocs

* Added a UrlPreparer and cleaned up some minor things

* Download entire archive and filter out the wanted files in readTree

* Added test for dir output of tree reader

* Fixed formatting

* Fixed formatting

* Some cleanups and fixes

* Fix typing issues

* Fix prettier

* Fix prettier

* Removed unused dependency

* Fixed comments on PR

* Prettier

* Moved @types/fs-extra to devDeps

* Fixed another PR comment

* Added a test for getDocFilesFromRepository

* Prettier

packages/backend-common/package.json

@@ -36,11 +36,13 @@
     "@types/cors": "^2.8.6",
     "@types/express": "^4.17.6",
     "compression": "^1.7.4",
+    "concat-stream": "^2.0.0",
     "cors": "^2.8.5",
     "cross-fetch": "^3.0.6",
     "express": "^4.17.1",
     "express-prom-bundle": "^6.1.0",
     "express-promise-router": "^3.0.3",
+    "fs-extra": "^9.0.1",
     "git-url-parse": "^11.4.0",
     "helmet": "^4.0.0",
     "knex": "^0.21.6",
@@ -51,6 +53,7 @@
     "prom-client": "^12.0.0",
     "selfsigned": "^1.10.7",
     "stoppable": "^1.1.0",
+    "tar": "^6.0.5",
     "winston": "^3.2.1"
   },
   "peerDependencies": {
@@ -64,17 +67,24 @@
   "devDependencies": {
     "@backstage/cli": "^0.2.0",
     "@types/compression": "^1.7.0",
+    "@types/concat-stream": "^1.6.0",
+    "@types/fs-extra": "^9.0.3",
     "@types/http-errors": "^1.6.3",
     "@types/minimist": "^1.2.0",
+    "@types/mock-fs": "^4.13.0",
     "@types/morgan": "^1.9.0",
+    "@types/recursive-readdir": "^2.2.0",
     "@types/stoppable": "^1.1.0",
     "@types/supertest": "^2.0.8",
+    "@types/tar": "^4.0.3",
     "@types/webpack-env": "^1.15.2",
     "@types/yaml": "^1.9.7",
     "get-port": "^5.1.1",
     "http-errors": "^1.7.3",
     "jest": "^26.0.1",
+    "mock-fs": "^4.13.0",
     "msw": "^0.21.2",
+    "recursive-readdir": "^2.2.2",
     "supertest": "^4.0.2"
   },
   "files": [

packages/backend-common/src/reading/GithubUrlReader.test.ts

@@ -15,6 +15,9 @@
  */
 
 import { ConfigReader } from '@backstage/config';
+import { setupServer } from 'msw/node';
+import { msw } from '@backstage/test-utils';
+import { rest } from 'msw';
 import {
   getApiRequestOptions,
   getApiUrl,
@@ -24,6 +27,10 @@ import {
   ProviderConfig,
   readConfig,
 } from './GithubUrlReader';
+import fs from 'fs';
+import path from 'path';
+import mockfs from 'mock-fs';
+import recursive from 'recursive-readdir';
 
 describe('GithubUrlReader', () => {
   describe('getApiRequestOptions', () => {
@@ -230,4 +237,77 @@ describe('GithubUrlReader', () => {
       );
     });
   });
+
+  describe('readTree', () => {
+    const worker = setupServer();
+
+    msw.setupDefaultHandlers(worker);
+
+    const repoBuffer = fs.readFileSync(
+      path.resolve('src', 'reading', '__fixtures__', 'repo.tar.gz'),
+    );
+
+    beforeEach(() => {
+      worker.use(
+        rest.get(
+          'https://github.com/spotify/mock/archive/repo.tar.gz',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/x-gzip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+    });
+
+    it('returns the wanted files from an archive', async () => {
+      const processor = new GithubUrlReader({
+        host: 'github.com',
+      });
+
+      const response = await processor.readTree(
+        'https://github.com/spotify/mock',
+        'repo',
+        ['mkdocs.yml', 'docs'],
+      );
+
+      const files = response.files();
+
+      const mkDocsFile = await files[0].content();
+      const indexMarkdownFile = await files[1].content();
+
+      expect(mkDocsFile.toString()).toBe('site_name: Test\n');
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('returns a folder path from an archive', async () => {
+      const processor = new GithubUrlReader({
+        host: 'github.com',
+      });
+
+      const response = await processor.readTree(
+        'https://github.com/spotify/mock',
+        'repo',
+        ['mkdocs.yml', 'docs'],
+      );
+
+      mockfs();
+      const directory = await response.dir('/tmp/fs');
+
+      const writtenToDirectory = fs.existsSync(directory);
+      const paths = await recursive(directory);
+      mockfs.restore();
+
+      expect(writtenToDirectory).toBe(true);
+      expect(paths.sort()).toEqual(
+        [
+          '/tmp/fs/mock-repo/docs/index.md',
+          '/tmp/fs/mock-repo/mkdocs.yml',
+        ].sort(),
+      );
+
+      worker.resetHandlers();
+    });
+  });
 });

packages/backend-common/src/reading/GithubUrlReader.ts

@@ -18,7 +18,12 @@ import { Config } from '@backstage/config';
 import parseGitUri from 'git-url-parse';
 import fetch from 'cross-fetch';
 import { NotFoundError } from '../errors';
-import { ReaderFactory, UrlReader } from './types';
+import { ReaderFactory, ReadTreeResponse, UrlReader, File } from './types';
+import tar from 'tar';
+import fs from 'fs-extra';
+import concatStream from 'concat-stream';
+import path from 'path';
+import os from 'os';
 
 /**
  * The configuration parameters for a single GitHub API provider.
@@ -229,6 +234,92 @@ export class GithubUrlReader implements UrlReader {
     throw new Error(message);
   }
 
+  private async getRepositoryArchive(
+    repoUrl: string,
+    branchName: string,
+  ): Promise<Response> {
+    return fetch(new URL(`${repoUrl}/archive/${branchName}.tar.gz`).toString());
+  }
+
+  private async writeBufferToFile(
+    filePath: string,
+    content: Buffer,
+  ): Promise<void> {
+    await fs.outputFile(filePath, content.toString());
+  }
+
+  async readTree(
+    repoUrl: string,
+    branchName: string,
+    paths: Array<string>,
+  ): Promise<ReadTreeResponse> {
+    const { name: repoName } = parseGitUri(repoUrl);
+
+    const repoArchive = await this.getRepositoryArchive(repoUrl, branchName);
+
+    const files: File[] = [];
+    return new Promise(resolve => {
+      const parser = new (tar.Parse as any)({
+        filter: (path: string) =>
+          !!paths.filter(file => {
+            return path.startsWith(`${repoName}-${branchName}/${file}`);
+          }).length,
+        onentry: (entry: tar.ReadEntry) => {
+          if (entry.type === 'Directory') {
+            entry.resume();
+            return;
+          }
+
+          const contentPromise: Promise<Buffer> = new Promise(res => {
+            entry.pipe(concatStream(res));
+          });
+
+          files.push({
+            path: entry.path,
+            content: () => contentPromise,
+          });
+
+          entry.resume();
+        },
+      });
+
+      // @ts-ignore Typescript doesn't consider .pipe a method on ReadableStream. Don't know why.
+      repoArchive.body?.pipe(parser).on('finish', () => {
+        resolve({
+          files: () => {
+            return files;
+          },
+          archive: () => {
+            return new Promise(resolve =>
+              resolve(Buffer.from('Archive is not yet implemented')),
+            );
+          },
+          dir: (outDir: string | undefined) => {
+            const targetDirectory =
+              outDir || fs.mkdtempSync(path.join(os.tmpdir(), 'backstage-'));
+
+            return new Promise((res, rej) => {
+              Promise.all(
+                files.map(async file => {
+                  return this.writeBufferToFile(
+                    `${targetDirectory}/${file.path}`,
+                    await file.content(),
+                  );
+                }),
+              )
+                .then(() => {
+                  res(`${targetDirectory}/${repoName}-${branchName}`);
+                })
+                .catch(err => {
+                  rej(err);
+                });
+            });
+          },
+        });
+      });
+    });
+  }
+
   toString() {
     const { host, token } = this.config;
     return `github{host=${host},authed=${Boolean(token)}}`;

packages/backend-common/src/reading/UrlReaderPredicateMux.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { UrlReader, UrlReaderPredicateTuple } from './types';
+import { ReadTreeResponse, UrlReader, UrlReaderPredicateTuple } from './types';
 
 type Options = {
   // UrlReader to fall back to if no other reader is matched
@@ -53,6 +53,33 @@ export class UrlReaderPredicateMux implements UrlReader {
     throw new Error(`No reader found that could handle '${url}'`);
   }
 
+  readTree(
+    repoUrl: string,
+    branchName: string,
+    paths: Array<string>,
+  ): Promise<ReadTreeResponse> {
+    const parsed = new URL(repoUrl);
+
+    for (const { predicate, reader } of this.readers) {
+      if (predicate(parsed)) {
+        if (reader.readTree) return reader.readTree(repoUrl, branchName, paths);
+        throw new Error(
+          `Trying to call readTree on UrlReader which does not support the feature.`,
+        );
+      }
+    }
+
+    if (this.fallback) {
+      if (this.fallback.readTree)
+        return this.fallback.readTree(repoUrl, branchName, paths);
+      throw new Error(
+        `Trying to call readTree on UrlReader which does not support the feature.`,
+      );
+    }
+
+    throw new Error(`No reader found that could handle '${repoUrl}'`);
+  }
+
   toString() {
     return `predicateMux{readers=${this.readers
       .map(t => t.reader)

packages/backend-common/src/reading/index.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-export type { UrlReader } from './types';
+export type { UrlReader, ReadTreeResponse } from './types';
 export { UrlReaders } from './UrlReaders';
 export { AzureUrlReader } from './AzureUrlReader';
 export { BitbucketUrlReader } from './BitbucketUrlReader';

packages/backend-common/src/reading/types.ts

@@ -22,6 +22,11 @@ import { Config } from '@backstage/config';
  */
 export type UrlReader = {
   read(url: string): Promise<Buffer>;
+  readTree?(
+    repoUrl: string,
+    branchName: string,
+    paths: Array<string>,
+  ): Promise<ReadTreeResponse>;
 };
 
 export type UrlReaderPredicateTuple = {
@@ -37,3 +42,14 @@ export type ReaderFactory = (options: {
   config: Config;
   logger: Logger;
 }) => UrlReaderPredicateTuple[];
+
+export type File = {
+  path: string;
+  content(): Promise<Buffer>;
+};
+
+export type ReadTreeResponse = {
+  files(): File[];
+  archive(): Promise<Buffer>;
+  dir(outDir?: string): Promise<string>;
+};

packages/backend/src/plugins/techdocs.ts

@@ -22,6 +22,7 @@ import {
   LocalPublish,
   TechdocsGenerator,
   CommonGitPreparer,
+  UrlPreparer,
 } from '@backstage/plugin-techdocs-backend';
 import { PluginEnvironment } from '../types';
 import Docker from 'dockerode';
@@ -30,20 +31,25 @@ export default async function createPlugin({
   logger,
   config,
   discovery,
+  reader,
 }: PluginEnvironment) {
   const generators = new Generators();
   const techdocsGenerator = new TechdocsGenerator(logger, config);
   generators.register('techdocs', techdocsGenerator);
 
   const preparers = new Preparers();
-  const commonGitPreparer = new CommonGitPreparer(logger);
 
   const directoryPreparer = new DirectoryPreparer(logger);
   preparers.register('dir', directoryPreparer);
+
+  const commonGitPreparer = new CommonGitPreparer(logger);
   preparers.register('github', commonGitPreparer);
   preparers.register('gitlab', commonGitPreparer);
   preparers.register('azure/api', commonGitPreparer);
 
+  const urlPreparer = new UrlPreparer(reader, logger);
+  preparers.register('url', urlPreparer);
+
   const publisher = new LocalPublish(logger);
 
   const dockerClient = new Docker();

plugins/techdocs-backend/src/helpers.test.ts

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getDocFilesFromRepository } from './helpers';
+import { UrlReader, ReadTreeResponse } from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+
+describe('getDocFilesFromRepository', () => {
+  it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in root', async () => {
+    class MockUrlReader implements UrlReader {
+      async read() {
+        return Buffer.from('mock');
+      }
+
+      async readTree(): Promise<ReadTreeResponse> {
+        return {
+          dir: async () => {
+            return '/tmp/testfolder';
+          },
+          files: () => {
+            return [];
+          },
+          archive: async () => {
+            return Buffer.from('');
+          },
+        };
+      }
+    }
+
+    const mockEntity: Entity = {
+      metadata: {
+        namespace: 'default',
+        annotations: {
+          'backstage.io/techdocs-ref':
+            'url:https://github.com/backstage/backstage/blob/master/mkdocs.yml',
+        },
+        name: 'mytestcomponent',
+        description: 'A component for testing',
+      },
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      spec: {
+        type: 'documentation',
+        lifecycle: 'experimental',
+        owner: 'testuser',
+      },
+    };
+
+    const output = await getDocFilesFromRepository(
+      new MockUrlReader(),
+      mockEntity,
+    );
+
+    expect(output).toBe('/tmp/testfolder/.');
+  });
+
+  it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in a subfolder', async () => {
+    class MockUrlReader implements UrlReader {
+      async read() {
+        return Buffer.from('mock');
+      }
+
+      async readTree(): Promise<ReadTreeResponse> {
+        return {
+          dir: async () => {
+            return '/tmp/testfolder';
+          },
+          files: () => {
+            return [];
+          },
+          archive: async () => {
+            return Buffer.from('');
+          },
+        };
+      }
+    }
+
+    const mockEntity: Entity = {
+      metadata: {
+        namespace: 'default',
+        annotations: {
+          'backstage.io/techdocs-ref':
+            'url:https://github.com/backstage/backstage/blob/master/subfolder/mkdocs.yml',
+        },
+        name: 'mytestcomponent',
+        description: 'A component for testing',
+      },
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      spec: {
+        type: 'documentation',
+        lifecycle: 'experimental',
+        owner: 'testuser',
+      },
+    };
+
+    const output = await getDocFilesFromRepository(
+      new MockUrlReader(),
+      mockEntity,
+    );
+
+    expect(output).toBe('/tmp/testfolder/subfolder');
+  });
+});

plugins/techdocs-backend/src/helpers.ts

@@ -22,7 +22,7 @@ import fs from 'fs-extra';
 import { getDefaultBranch } from './default-branch';
 import { getGitRepoType, getTokenForGitRepo } from './git-auth';
 import { Entity } from '@backstage/catalog-model';
-import { InputError } from '@backstage/backend-common';
+import { InputError, UrlReader } from '@backstage/backend-common';
 import { RemoteProtocol } from './techdocs/stages/prepare/types';
 import { Logger } from 'winston';
 
@@ -78,6 +78,7 @@ export const getLocationForEntity = (
     case 'github':
     case 'gitlab':
     case 'azure/api':
+    case 'url':
       return { type, target };
     case 'dir':
       if (path.isAbsolute(target)) return { type, target };
@@ -168,3 +169,34 @@ export const getLastCommitTimestamp = async (
 
   return commit.date().getTime();
 };
+
+export const getDocFilesFromRepository = async (
+  reader: UrlReader,
+  entity: Entity,
+): Promise<any> => {
+  const { target } = parseReferenceAnnotation(
+    'backstage.io/techdocs-ref',
+    entity,
+  );
+
+  const { ref, filepath: mkdocsPath } = parseGitUrl(target);
+
+  const docsRootPath = path.dirname(mkdocsPath);
+  const docsFolderPath = path.join(docsRootPath, 'docs');
+
+  if (reader.readTree) {
+    const readTreeResponse = await reader.readTree(
+      parseGitUrl(target).toString(),
+      ref,
+      [mkdocsPath, docsFolderPath],
+    );
+
+    const tmpDir = await readTreeResponse.dir();
+
+    return `${tmpDir}/${docsRootPath}`;
+  }
+
+  throw new Error(
+    `No readTree method available on the UrlReader for ${target}`,
+  );
+};

plugins/techdocs-backend/src/service/helpers.ts

@@ -103,7 +103,7 @@ export class DocsBuilder {
     const { type, target } = getLocationForEntity(this.entity);
 
     // Unless docs are stored locally
-    const nonAgeCheckTypes = ['dir', 'file'];
+    const nonAgeCheckTypes = ['dir', 'file', 'url'];
     if (!nonAgeCheckTypes.includes(type)) {
       const lastCommit = await getLastCommitTimestamp(target, this.logger);
       const storageTimeStamp = buildMetadataStorage.getTimestamp();

plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts

@@ -15,5 +15,6 @@
  */
 export { DirectoryPreparer } from './dir';
 export { CommonGitPreparer } from './commonGit';
+export { UrlPreparer } from './url';
 export { Preparers } from './preparers';
 export type { PreparerBuilder, PreparerBase } from './types';

plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts

@@ -30,4 +30,10 @@ export type PreparerBuilder = {
   get(entity: Entity): PreparerBase;
 };
 
-export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file' | 'azure/api';
+export type RemoteProtocol =
+  | 'dir'
+  | 'github'
+  | 'gitlab'
+  | 'file'
+  | 'azure/api'
+  | 'url';

plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity } from '@backstage/catalog-model';
+import { PreparerBase } from './types';
+import { getDocFilesFromRepository } from '../../../helpers';
+
+import { Logger } from 'winston';
+import { UrlReader } from '@backstage/backend-common';
+
+export class UrlPreparer implements PreparerBase {
+  private readonly logger: Logger;
+  private readonly reader: UrlReader;
+
+  constructor(reader: UrlReader, logger: Logger) {
+    this.logger = logger;
+    this.reader = reader;
+  }
+
+  async prepare(entity: Entity): Promise<string> {
+    try {
+      return getDocFilesFromRepository(this.reader, entity);
+    } catch (error) {
+      this.logger.debug(
+        `Unable to fetch files for building docs ${error.message}`,
+      );
+      throw error;
+    }
+  }
+}

yarn.lock

@@ -4916,6 +4916,13 @@
   dependencies:
     "@types/express" "*"
 
+"@types/concat-stream@^1.6.0":
+  version "1.6.0"
+  resolved "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.0.tgz#394dbe0bb5fee46b38d896735e8b68ef2390d00d"
+  integrity sha1-OU2+C7X+5Gs42JZzXoto7yOQ0A0=
+  dependencies:
+    "@types/node" "*"
+
 "@types/connect-history-api-fallback@*":
   version "1.3.3"
   resolved "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.3.3.tgz#4772b79b8b53185f0f4c9deab09236baf76ee3b4"
@@ -5109,6 +5116,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/fs-extra@^9.0.3":
+  version "9.0.3"
+  resolved "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.3.tgz#9996e5cce993508c32325380b429f04a1327523e"
+  integrity sha512-NKdGoXLTFTRED3ENcfCsH8+ekV4gbsysanx2OPbstXVV6fZMgUCqTxubs6I9r7pbOJbFgVq1rpFtLURjKCZWUw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/git-url-parse@^9.0.0":
   version "9.0.0"
   resolved "https://registry.npmjs.org/@types/git-url-parse/-/git-url-parse-9.0.0.tgz#aac1315a44fa4ed5a52c3820f6c3c2fb79cbd12d"
@@ -22456,7 +22470,7 @@ tar@^4, tar@^4.4.10, tar@^4.4.12, tar@^4.4.8:
     safe-buffer "^5.1.2"
     yallist "^3.0.3"
 
-tar@^6.0.1, tar@^6.0.2:
+tar@^6.0.1, tar@^6.0.2, tar@^6.0.5:
   version "6.0.5"
   resolved "https://registry.npmjs.org/tar/-/tar-6.0.5.tgz#bde815086e10b39f1dcd298e89d596e1535e200f"
   integrity sha512-0b4HOimQHj9nXNEAA7zWwMM91Zhhba3pspja6sQbgTpynOJf+bkjBnfybNYzbpLbnwXnbyB4LOREvlyXLkCHSg==