backend-common: remove hardcoded size check and replace with filter param

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

packages/backend-common/src/reading/tree/TarArchiveResponse.ts

@@ -43,7 +43,7 @@ export class TarArchiveResponse implements ReadTreeResponse {
     private readonly subPath: string,
     private readonly workDir: string,
     public readonly etag: string,
-    private readonly filter?: (path: string) => boolean,
+    private readonly filter?: (path: string, info: { size: number }) => boolean,
   ) {
     if (subPath) {
       if (!subPath.endsWith('/')) {
@@ -92,17 +92,12 @@ export class TarArchiveResponse implements ReadTreeResponse {
 
       const path = relativePath.slice(this.subPath.length);
       if (this.filter) {
-        if (!this.filter(path)) {
+        if (!this.filter(path, { size: entry.remain })) {
           entry.resume();
           return;
         }
       }
 
-      if (entry.size && entry.size >= 20000) {
-        entry.resume();
-        return;
-      }
-
       const content = new Promise<Buffer>(async resolve => {
         await pipeline(entry, concatStream(resolve));
       });
@@ -160,7 +155,7 @@ export class TarArchiveResponse implements ReadTreeResponse {
       tar.extract({
         strip,
         cwd: dir,
-        filter: path => {
+        filter: (path, stat) => {
           // File path relative to the root extracted directory. Will remove the
           // top level dir name from the path since its name is hard to predetermine.
           const relativePath = stripFirstDirectoryFromPath(path);
@@ -169,7 +164,7 @@ export class TarArchiveResponse implements ReadTreeResponse {
           }
           if (this.filter) {
             const innerPath = path.split('/').slice(strip).join('/');
-            return this.filter(innerPath);
+            return this.filter(innerPath, { size: stat.size });
           }
           return true;
         },

packages/backend-common/src/reading/tree/ZipArchiveResponse.ts

@@ -37,7 +37,7 @@ export class ZipArchiveResponse implements ReadTreeResponse {
     private readonly subPath: string,
     private readonly workDir: string,
     public readonly etag: string,
-    private readonly filter?: (path: string) => boolean,
+    private readonly filter?: (path: string, info: { size: number }) => boolean,
   ) {
     if (subPath) {
       if (!subPath.endsWith('/')) {
@@ -68,11 +68,6 @@ export class ZipArchiveResponse implements ReadTreeResponse {
 
   private shouldBeIncluded(entry: Entry): boolean {
     const strippedPath = stripFirstDirectoryFromPath(entry.path);
-    const size = entry.vars.compressedSize;
-
-    if (size >= 20000) {
-      return false;
-    }
 
     if (this.subPath) {
       if (!strippedPath.startsWith(this.subPath)) {
@@ -80,7 +75,11 @@ export class ZipArchiveResponse implements ReadTreeResponse {
       }
     }
     if (this.filter) {
-      return this.filter(this.getInnerPath(entry.path));
+      return this.filter(this.getInnerPath(entry.path), {
+        size:
+          (entry.vars as { uncompressedSize?: number }).uncompressedSize ??
+          entry.vars.compressedSize,
+      });
     }
     return true;
   }

packages/backend-common/src/reading/types.ts

@@ -104,7 +104,7 @@ export type ReadTreeOptions = {
    *
    * If no filter is provided all files are extracted.
    */
-  filter?(path: string): boolean;
+  filter?(path: string, info?: { size: number }): boolean;
 
   /**
    * An etag can be provided to check whether readTree's response has changed from a previous execution.
@@ -164,7 +164,7 @@ export type FromArchiveOptions = {
   // etag of the blob
   etag: string;
   // Filter passed on from the ReadTreeOptions
-  filter?: (path: string) => boolean;
+  filter?: (path: string, info?: { size: number }) => boolean;
 };
 
 export interface ReadTreeResponseFactory {