diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index adcf420fc0..f7ab7f5349 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -43,7 +43,7 @@ export class TarArchiveResponse implements ReadTreeResponse { private readonly subPath: string, private readonly workDir: string, public readonly etag: string, - private readonly filter?: (path: string) => boolean, + private readonly filter?: (path: string, info: { size: number }) => boolean, ) { if (subPath) { if (!subPath.endsWith('/')) { @@ -92,17 +92,12 @@ export class TarArchiveResponse implements ReadTreeResponse { const path = relativePath.slice(this.subPath.length); if (this.filter) { - if (!this.filter(path)) { + if (!this.filter(path, { size: entry.remain })) { entry.resume(); return; } } - if (entry.size && entry.size >= 20000) { - entry.resume(); - return; - } - const content = new Promise(async resolve => { await pipeline(entry, concatStream(resolve)); }); @@ -160,7 +155,7 @@ export class TarArchiveResponse implements ReadTreeResponse { tar.extract({ strip, cwd: dir, - filter: path => { + filter: (path, stat) => { // File path relative to the root extracted directory. Will remove the // top level dir name from the path since its name is hard to predetermine. const relativePath = stripFirstDirectoryFromPath(path); @@ -169,7 +164,7 @@ export class TarArchiveResponse implements ReadTreeResponse { } if (this.filter) { const innerPath = path.split('/').slice(strip).join('/'); - return this.filter(innerPath); + return this.filter(innerPath, { size: stat.size }); } return true; }, diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 62219ec27d..5c12a383ee 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -37,7 +37,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { private readonly subPath: string, private readonly workDir: string, public readonly etag: string, - private readonly filter?: (path: string) => boolean, + private readonly filter?: (path: string, info: { size: number }) => boolean, ) { if (subPath) { if (!subPath.endsWith('/')) { @@ -68,11 +68,6 @@ export class ZipArchiveResponse implements ReadTreeResponse { private shouldBeIncluded(entry: Entry): boolean { const strippedPath = stripFirstDirectoryFromPath(entry.path); - const size = entry.vars.compressedSize; - - if (size >= 20000) { - return false; - } if (this.subPath) { if (!strippedPath.startsWith(this.subPath)) { @@ -80,7 +75,11 @@ export class ZipArchiveResponse implements ReadTreeResponse { } } if (this.filter) { - return this.filter(this.getInnerPath(entry.path)); + return this.filter(this.getInnerPath(entry.path), { + size: + (entry.vars as { uncompressedSize?: number }).uncompressedSize ?? + entry.vars.compressedSize, + }); } return true; } diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 8efc833ead..93f287d4fe 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -104,7 +104,7 @@ export type ReadTreeOptions = { * * If no filter is provided all files are extracted. */ - filter?(path: string): boolean; + filter?(path: string, info?: { size: number }): boolean; /** * An etag can be provided to check whether readTree's response has changed from a previous execution. @@ -164,7 +164,7 @@ export type FromArchiveOptions = { // etag of the blob etag: string; // Filter passed on from the ReadTreeOptions - filter?: (path: string) => boolean; + filter?: (path: string, info?: { size: number }) => boolean; }; export interface ReadTreeResponseFactory {