readTree method on UrlReader (#2925)

* Basic implementation of readTree on github and use it in techdocs

* Added a UrlPreparer and cleaned up some minor things

* Download entire archive and filter out the wanted files in readTree

* Added test for dir output of tree reader

* Fixed formatting

* Fixed formatting

* Some cleanups and fixes

* Fix typing issues

* Fix prettier

* Fix prettier

* Removed unused dependency

* Fixed comments on PR

* Prettier

* Moved @types/fs-extra to devDeps

* Fixed another PR comment

* Added a test for getDocFilesFromRepository

* Prettier

packages/backend-common/package.json

@@ -36,11 +36,13 @@
     "@types/cors": "^2.8.6",
     "@types/express": "^4.17.6",
     "compression": "^1.7.4",
+    "concat-stream": "^2.0.0",
     "cors": "^2.8.5",
     "cross-fetch": "^3.0.6",
     "express": "^4.17.1",
     "express-prom-bundle": "^6.1.0",
     "express-promise-router": "^3.0.3",
+    "fs-extra": "^9.0.1",
     "git-url-parse": "^11.4.0",
     "helmet": "^4.0.0",
     "knex": "^0.21.6",
@@ -51,6 +53,7 @@
     "prom-client": "^12.0.0",
     "selfsigned": "^1.10.7",
     "stoppable": "^1.1.0",
+    "tar": "^6.0.5",
     "winston": "^3.2.1"
   },
   "peerDependencies": {
@@ -64,17 +67,24 @@
   "devDependencies": {
     "@backstage/cli": "^0.2.0",
     "@types/compression": "^1.7.0",
+    "@types/concat-stream": "^1.6.0",
+    "@types/fs-extra": "^9.0.3",
     "@types/http-errors": "^1.6.3",
     "@types/minimist": "^1.2.0",
+    "@types/mock-fs": "^4.13.0",
     "@types/morgan": "^1.9.0",
+    "@types/recursive-readdir": "^2.2.0",
     "@types/stoppable": "^1.1.0",
     "@types/supertest": "^2.0.8",
+    "@types/tar": "^4.0.3",
     "@types/webpack-env": "^1.15.2",
     "@types/yaml": "^1.9.7",
     "get-port": "^5.1.1",
     "http-errors": "^1.7.3",
     "jest": "^26.0.1",
+    "mock-fs": "^4.13.0",
     "msw": "^0.21.2",
+    "recursive-readdir": "^2.2.2",
     "supertest": "^4.0.2"
   },
   "files": [