From cd973a35d8b18671a12062478d39a5bf549fcad8 Mon Sep 17 00:00:00 2001 From: Otto Sichert Date: Wed, 15 Jun 2022 11:23:54 +0200 Subject: [PATCH 01/20] Implement manual catching of corrupted ZIP files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Emma Indal Co-authored-by: Anders Näsman Co-authored-by: Raghunandan Balachandran Signed-off-by: Otto Sichert --- .../src/reading/tree/ZipArchiveResponse.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index e07cedde87..6505ef4d20 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -86,9 +86,22 @@ export class ZipArchiveResponse implements ReadTreeResponse { const files = Array(); - await this.stream - .pipe(unzipper.Parse()) + // Some corrupted ZIP files cause the zlib inflater to hang indefinitely. + // This is a workaround to bail on stuck streams after 3 seconds. + const piped = this.stream.pipe(unzipper.Parse()); + let lastEntryTimeout: NodeJS.Timeout | undefined; + + await piped .on('entry', (entry: Entry) => { + clearTimeout(lastEntryTimeout); + + lastEntryTimeout = setTimeout(() => { + piped.emit( + 'error', + new Error(`Timed out unzipping file ${entry.path}`), + ); + }, 3000); + if (entry.type === 'Directory') { entry.resume(); return; @@ -105,6 +118,8 @@ export class ZipArchiveResponse implements ReadTreeResponse { }) .promise(); + clearTimeout(lastEntryTimeout); + return files; } From 67e2671e23d2b4f830c7d4823a930765cbfbd9f5 Mon Sep 17 00:00:00 2001 From: Otto Sichert Date: Wed, 15 Jun 2022 12:16:32 +0200 Subject: [PATCH 02/20] Move stream timeout promise to util Signed-off-by: Otto Sichert --- .../src/reading/tree/ZipArchiveResponse.ts | 33 ++++++------------- .../backend-common/src/reading/tree/util.ts | 27 +++++++++++++++ 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 6505ef4d20..50972a52a4 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -24,6 +24,7 @@ import { ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; +import { streamToTimeoutPromise } from './util'; /** * Wraps a zip archive stream into a tree response reader. @@ -86,27 +87,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { const files = Array(); - // Some corrupted ZIP files cause the zlib inflater to hang indefinitely. - // This is a workaround to bail on stuck streams after 3 seconds. - const piped = this.stream.pipe(unzipper.Parse()); - let lastEntryTimeout: NodeJS.Timeout | undefined; - - await piped + const parseStream = this.stream + .pipe(unzipper.Parse()) .on('entry', (entry: Entry) => { - clearTimeout(lastEntryTimeout); - - lastEntryTimeout = setTimeout(() => { - piped.emit( - 'error', - new Error(`Timed out unzipping file ${entry.path}`), - ); - }, 3000); - - if (entry.type === 'Directory') { - entry.resume(); - return; - } - if (this.shouldBeIncluded(entry)) { files.push({ path: this.getInnerPath(entry.path), @@ -115,10 +98,14 @@ export class ZipArchiveResponse implements ReadTreeResponse { } else { entry.autodrain(); } - }) - .promise(); + }); - clearTimeout(lastEntryTimeout); + await streamToTimeoutPromise(parseStream, { + eventName: 'entry', + timeoutMs: 3000, + getError: (entry: Entry) => + new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`), + }); return files; } diff --git a/packages/backend-common/src/reading/tree/util.ts b/packages/backend-common/src/reading/tree/util.ts index cfb986a0b0..51f904f313 100644 --- a/packages/backend-common/src/reading/tree/util.ts +++ b/packages/backend-common/src/reading/tree/util.ts @@ -23,3 +23,30 @@ const directoryNameRegex = /^[^\/]+\//; export function stripFirstDirectoryFromPath(path: string): string { return path.replace(directoryNameRegex, ''); } + +// Some corrupted ZIP files cause the zlib inflater to hang indefinitely. +// This is a workaround to bail on stuck streams after 3 seconds. +export async function streamToTimeoutPromise( + stream: NodeJS.ReadableStream, + options: { + timeoutMs: number; + eventName: string; + getError: (data: T) => Error; + }, +) { + let lastEntryTimeout: NodeJS.Timeout | undefined; + stream.on(options.eventName, (data: T) => { + clearTimeout(lastEntryTimeout); + + lastEntryTimeout = setTimeout(() => { + stream.emit('error', options.getError(data)); + }, options.timeoutMs); + }); + + await new Promise(function (resolve, reject) { + stream.on('finish', resolve); + stream.on('error', reject); + }); + + clearTimeout(lastEntryTimeout); +} From a6a11e52ef88aadb2b6cc5a7b640192564e85d2d Mon Sep 17 00:00:00 2001 From: Otto Sichert Date: Wed, 15 Jun 2022 12:27:43 +0200 Subject: [PATCH 03/20] Ensure all ZIP operations handle timeouts correctly Signed-off-by: Otto Sichert --- .../src/reading/tree/ZipArchiveResponse.ts | 25 ++++++++++++++----- .../backend-common/src/reading/tree/util.ts | 1 + 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 50972a52a4..ffc2c8d9a7 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -118,7 +118,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { } const archive = archiver('zip'); - await this.stream + const parseStream = this.stream .pipe(unzipper.Parse()) .on('entry', (entry: Entry) => { if (entry.type === 'File' && this.shouldBeIncluded(entry)) { @@ -126,8 +126,15 @@ export class ZipArchiveResponse implements ReadTreeResponse { } else { entry.autodrain(); } - }) - .promise(); + }); + + await streamToTimeoutPromise(parseStream, { + eventName: 'entry', + timeoutMs: 3000, + getError: (entry: Entry) => + new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`), + }); + archive.finalize(); return archive; @@ -140,7 +147,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { options?.targetDir ?? (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); - await this.stream + const parseStream = this.stream .pipe(unzipper.Parse()) .on('entry', async (entry: Entry) => { // Ignore directory entries since we handle that with the file entries @@ -155,8 +162,14 @@ export class ZipArchiveResponse implements ReadTreeResponse { } else { entry.autodrain(); } - }) - .promise(); + }); + + await streamToTimeoutPromise(parseStream, { + eventName: 'entry', + timeoutMs: 3000, + getError: (entry: Entry) => + new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`), + }); return dir; } diff --git a/packages/backend-common/src/reading/tree/util.ts b/packages/backend-common/src/reading/tree/util.ts index 51f904f313..1d7a42c93e 100644 --- a/packages/backend-common/src/reading/tree/util.ts +++ b/packages/backend-common/src/reading/tree/util.ts @@ -26,6 +26,7 @@ export function stripFirstDirectoryFromPath(path: string): string { // Some corrupted ZIP files cause the zlib inflater to hang indefinitely. // This is a workaround to bail on stuck streams after 3 seconds. +// Related: https://github.com/ZJONSSON/node-unzipper/issues/213 export async function streamToTimeoutPromise( stream: NodeJS.ReadableStream, options: { From 72e34fe980ec46be3053201ff5b8075997c92b60 Mon Sep 17 00:00:00 2001 From: Otto Sichert Date: Thu, 16 Jun 2022 00:13:31 +0200 Subject: [PATCH 04/20] Restore original behaviour in listing ZIP directories Signed-off-by: Otto Sichert --- .../backend-common/src/reading/tree/ZipArchiveResponse.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index ffc2c8d9a7..418f32b155 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -90,6 +90,11 @@ export class ZipArchiveResponse implements ReadTreeResponse { const parseStream = this.stream .pipe(unzipper.Parse()) .on('entry', (entry: Entry) => { + if (entry.type === 'Directory') { + entry.resume(); + return; + } + if (this.shouldBeIncluded(entry)) { files.push({ path: this.getInnerPath(entry.path), From 9344fdf1b67e551b63412b2df9ce16f390c0c6fa Mon Sep 17 00:00:00 2001 From: Otto Sichert Date: Thu, 16 Jun 2022 00:40:26 +0200 Subject: [PATCH 05/20] Simplify guarding against corrupt ZIP files Signed-off-by: Otto Sichert --- .../src/reading/tree/ZipArchiveResponse.ts | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 418f32b155..501113d9f1 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -26,6 +26,14 @@ import { } from '../types'; import { streamToTimeoutPromise } from './util'; +const guardCorruptZipStream = (stream: Readable) => + streamToTimeoutPromise(stream, { + eventName: 'entry', + timeoutMs: 3000, + getError: (entry: Entry) => + new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`), + }); + /** * Wraps a zip archive stream into a tree response reader. */ @@ -104,13 +112,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { entry.autodrain(); } }); - - await streamToTimeoutPromise(parseStream, { - eventName: 'entry', - timeoutMs: 3000, - getError: (entry: Entry) => - new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`), - }); + await guardCorruptZipStream(parseStream); return files; } @@ -132,13 +134,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { entry.autodrain(); } }); - - await streamToTimeoutPromise(parseStream, { - eventName: 'entry', - timeoutMs: 3000, - getError: (entry: Entry) => - new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`), - }); + await guardCorruptZipStream(parseStream); archive.finalize(); @@ -168,13 +164,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { entry.autodrain(); } }); - - await streamToTimeoutPromise(parseStream, { - eventName: 'entry', - timeoutMs: 3000, - getError: (entry: Entry) => - new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`), - }); + await guardCorruptZipStream(parseStream); return dir; } From b19d82600d6f2d5a66e854d863dd79fad1a44529 Mon Sep 17 00:00:00 2001 From: Otto Sichert Date: Thu, 16 Jun 2022 14:49:18 +0200 Subject: [PATCH 06/20] Add tests for corrupted ZIP files Signed-off-by: Otto Sichert --- .../src/reading/__fixtures__/mock-corrupted.zip | Bin 0 -> 34594 bytes .../src/reading/tree/ZipArchiveResponse.test.ts | 15 +++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 packages/backend-common/src/reading/__fixtures__/mock-corrupted.zip diff --git a/packages/backend-common/src/reading/__fixtures__/mock-corrupted.zip b/packages/backend-common/src/reading/__fixtures__/mock-corrupted.zip new file mode 100644 index 0000000000000000000000000000000000000000..51c4c268c7c1f7b0ea240777eb602af2fffe826e GIT binary patch literal 34594 zcmZshV{m3cyRPGjZD*p1ZQHhO+sRC9+qNe58z*mU+qQY;+XuU9*XdPvudY?ySKm+n z>hAR@%78<_g8Zk{`{^h>hF^7a zo^T+b5Ko{WAQ0&Pdb(J-ni<#|*_m+@tD3pE!Yckh=7Km-^FC1e= zfid4d{SVxK4{!NrDHVNM^T$8He}wRVhg;d3nt9UOnf@0h{r~TZ^naQu|7XUHo|QQx z7zE@$LjAuq69;E!H%C`9Q+jVJ$NxfZ{0s6w`u{llPrx7`qkkVv*yybt%`E@|Ka|(ka0|^Gj*CU~bW{FvOCz-Rt63$1~`z<|W-e*KQpTGOWs7(@dB`6CJ{ z4FVzwii4{B`?i71GXvRS^qUP_m43c^oXkZ+Tk9~kA+o9elE*{!;JSQ63G9o;6a)tt zu&&=OOIG|%*ZFaN8<0$A^}T+#_sxn3b_3ETEbkN&_%VhNNuyDxUr=0HR9aeCXkOec zBr?)JIyTlnIx#Ut?iiI56ql6o+VWz@FGPZ(gwmpe>OmHK4GkRy{e2#MaJE>rkh}oD zumInGF>%rVop-oLCUxPm8`n3 zOpm8J4_ZuSYFd0?ijID8kk;In_6YJS2kT8}HAw@{*jd#jDV~vboA>MWDy@~N??=n4 z*(KjVd3lDG=IRP0L`blh=-~kddP=eyA@9Q6(k$d3gqUbK*~tkyeEghTP64qtPp|iv zx7Q!}`XPW=&-*7mGTfy2^yCi`ayGoIq^vXpMNL&DEj2wA9gXdqnol1Ii)AF;XJUU# zPg7T0Ut@QBe;eQ!1^9grn&OX zF!|*$^p&QM27-VCJlspzgt!7*-z|I83EfBg{6l@&>x7R_-o@Zh+)FILdi$ugh?3+G ztPw=wbbYQ>m!^bS=k2<`_!(NPXl-00k~>sM{-R<+yJ5G9) zh{S?g!u<@WK8vtRZ)QZWK?tajOP{|y15OBqFtQcM_R;U86uF|g2Sm+1Ea976gSpIl zWs=-hCQCBm%;80g&-Uj@x+n2NeA+)aXwLDn`$a!PNN`A77ZopvnCWiYIcV#Gd>ju* z|M+Yb$4$XT2=;p=@=xX_pn0jaS9}Q?30YdZbD-%W#n;bj!n|_~Rb4C#u=G48zIi36 z&XdUKIXT~|DaNN_mfxi#1ymM|*pX%$aj!cO?i@-V8;n5?p&rrcc4&R}5cd#qjF-N6 zu=OWGwKdmx{5nbtPMPwTan7e1lvB@E_J>~N`7p!Pnl!~td^4q21;}7ZUn%%8~5Fj0PRCJtue{7uVJ1Q5jvZ;IsR?#0*EkI{Y3rO>+cXVS8Dd)CbrsO2dk zlk{>-I+l{Nt8PFfTBSJ&%4?L;M_<3>8A~EPX93ocCaYoJlGl>9#U{!+%!~$Qct*a7H-pKOnp3pmeuS`_=sJ}TcrZ`WTdfZ z@JyQJ0mJ5sm)~-_T!um~?S@UCm;M*P=~c(;c=cGZw)89l)67N66LG)n%hbbaQzt(%Y~9T`gk(35I~pvB5Upur2Yc$7H+ z+Hq7vU7u5vpJ&2BdOy)o<|+1dET`SUfWmBmF;Y#W6@!jbt}!%_*Z6xF5m z*T`}))>vYQMo(zv#}cln80OEYVNp@C(DE{UkDs)FUmcGYmF-7@0w{1BTp#{TMBPuF zGBj~8-Ubm`VEDp(b(TjtTqE)y_5lK7Gh!;-;27EKxyH1OvT_(In}BbBF#M!ZgOzLdV_&!` zcS9Yk)H70Cgj4M=^SdA;9=lHAnrCiyiPn83V(RAZzxJJ{K50KNKAA_VwAYzm3$|4_ zl48UFr_inbcKSX9Smqk@_?;IkoHvp9o;Af?hToXgjQDlIHlol~_!}>~|rxrYDkN~|$Zb&;( z9h7-J;AHZ~xwzj}he)WZOM+{wDas#m@+kqwrh-gypZ5S>;u@8YL&}9oSVsR6nAisO z9+uhE)<;nDV;6_?c+$?Z?5z>de$>^+3O_x1CuR>PM2U<`IoNg>feACoqxM~0f~Bex^yrW2hlJrpCNYmpUj-INHVXKV zQ)$yFKHEV4C&DW5d7i$)13L^k^;R}Ja5vKvZ)!m2Zw?IrH+d($X3)zS7%~wbyVN5UZ~}1GeG#v2Mn88xOlB?;m0=5H$$h9G z_h&oE-}Ef5^?Xu;YcuUxIzQ8{Z}KofK%kA>VhmcsHA7O?Pac@!cO;-U;=L3}(*crg ztoFmHldFdMUIJ=#e|M^Fi~3FD1b+x;0h0J#QZvx{RfJp5af8VOJS$FmDW=)~c^kd0 zsL8yO

rcvNb! z=XkDT7;S8&_b~&hXzFQSZ(Z2uQ#xW#V89oi0*(1~HeOK^OO~4{zSVj=shgDpqQxZO zBp9%rR;SLyC)>B$6bl%?F>KuCq;!temT`X8!}!?-)c)MO_{Dz^a9WD^igEW%{&qDHN9NJ3QVgMQ* z)u{S()x8Ue05e{Ik4R`*Uad?n%jz*EQRsd5y)aYwX%e#`jVYwD9mTE zHWoDEHN+r-@!@6Z3Ff1~9Vi-foW)_PJW2ols~{YW7j(fHYnQ}b3*u3Atuo}dai3^J zdThsqZHZ<9OapYY(A#>Rj1IHT*cg!fTcU-cEe;e(2~DlzGDV>b)&2C00fOSa9wHd8-0oNzTAFXCctaPR0WKmiDT0jy z{7M`8*mY#n1MNaDe0uh%eXN_&I-SKIzP2Rs>Chx4NbB;wvdo3btzQ>Tgf_`A0QM!+ zP+OMa8CzmYH~KqFT%TF|4R0Ce^z=k49b2p2%(IP)t9@~Z{MAhG`GJ`Gxv7~@%X3wL z*_~x}&fWL?;0Wu5`J&`&KmFijhpa9>P3XHIL6(+w<6@(dm$y0oNdnuV&`s&81%|VKu3Y%~GiLbc+@W@#D|ENl2SJh16>IaShBhGrRv>poKiuDwSS1$AnwCPBMh5N zhQnz6q`0jQ!A*%^4$3uoGh61}3>*BRYm04*}9je8`?OK$sQb+=8LWQlDh?Ch4J6 zFWya+-7I{V#`z@m=mAdIbsPNZ2)BO1+y-Z&YkkJ8Kmvk6ZI(63swZDFheoNOpOAkx zpodPUF}{pOsZW55nfN=2D|pw7n0#M&Hf%lNIMGuIY!8#!g>wonGo(wt;#YWMc!%g= zUO{{cdn0W5VCX(SWPLldz1mGkQ$vO3GGAra;ma>xPo3$%_z=vJXQ*k(|o0&1Cbl_+(`OB;P;>ihML`GDR(lrEHp6n*S8)WFNV>RmI{nAMj0NDghIm8p9B`MEsR$Smw~5GD zlcOHa=nF&M>&M4*nBm^X{fdk7*`m>M;3uL zD9k}0X)s0fVNg%Db`G$!Lgzi)&I-Ett!Z(H?PxvQen7(duc#0t&)XeXD&V?gB35*o zxug{mh=$~!II`YnABGE7w?88t2B3F5y%He#?_>AB4tKE-$bDKEcpdy0VxHI;MdZbC zpEBaNd#*R6y)5W=)L&i6nm3^7A0lMIE2p-vyY}q)$2$@EH%UwkXk3rZT%On%;@zaG z!}6LB9(ar~VY?NUqj zy~)RFK*4d9Wl@zt$GAU$2~v0+j`JH-wycz$Zi5s$c}^Aw&dkP@>HWfsOx==mpBz*b z`vG@f=Z+m0_2BMVD|31c<*&pU5%!W2c1%-cA>Sh-=+JZV&7ND`8hgV#nLUZA^V`g} ziK%T_R&>x6(;fQ^Bj~XNmEU7*q2~KHrK-1mPjz>kiB^y{zhpWvcSE~&T6He>x92h^ z8&&e;@Ox=vyU%_E+RwrwIpBdbs(A%}pTz}n)$|_N8S}d_1w6{$>2&EO*V#>!rR@je z@~a250)ghf$bJiTq?Mm)K12f--&8yTpVU_=dDSJ1noGN0REwf$8-Yvfh?(|>|`dwsEaZ*^}@fiHKrc(s9saMsw zU%bu<0MVOUpW@e$U5i?e_A)zKcqBr-%&-9nTtfG4hAm&!=0p6gbh(`Jmupr2FOhpk z!%TKQ*5~s9VY+tOK7NW??!ylaFq?X*ST0svhGK&Ul99*el;%OJ%7lGI-$0RC%sVha zN^b0udY!A|m`DN^ix;#bVuQJ=N;*h{0%PxfJsTyws5!)NZFB9%+3l?5=Dg`c*3hhC z3T3w9zG^b8qj|Ksyaj6$!Iw6>Io@guAQeDV`C3&61@{s!H&^Xqct98wSJ2%2Bb#I` zn+fQG!!i)MjczMBcf_s=skrOnYyN)E$bum!T3C(l3?V6|5pdX1G8>gxN@6A?Z9v$N zR#u-=>xg^^*YU)VtRc#9`AU9hGAGknO5*pOxBg8nD1lKEmJ?v)e24*3uN(lW$@a?; zsovXskST0R4`cCi)nF3bYRhNZ1B{{!bw6DI9Y#u9W!D!AT(lc@3CVROD^_J1HWjhV zuKqQePAvD2u}P4AT(6W2yp|{Q$}et#5+rqLD<-JN+8!pUhRr!v)afiDd`-m1t3B#-W{$;q1Tzd5P!Kk_2y z&IK@8Sne!%L)SX)t;@hn>xzhK?+H!*iY$Eb&5pDcC!XmH_Y+EIYmrPx22n~<(f z;a5fZOE*}#K}+%pj~;blrgysf*rTibmkTMx~g?$%>PPJ!ps2?Qz@GF(=|0d z38~d~q7EyiVN_d#l4)>Ap)z#p{lbo$Jw}C!3*;AGAz5~6ak3u66QtO3#dnbMPtt0Eg z0fmY~Ccw{fBR zq-2ce!%2n9;_L0Qds0}wVRTYf*#gSFEj*mRhO>FAA`jRy5hr?Piu)^t=b0k9)scG4 zdHlPq&T6oz#)KFtR&#Ej=Yj>t|WNijD>GGi+0sX(AfQ`*X_D9n9JUHa`lN- zjk`$o8Rn34h?(fZ4DF_1561^;r@Sw4OH7ti{ozl|N?Y2N>j&~vnE>@w;mvCj z9)croj(SYUwmS4y1=#M0(vq8X&*@a<9yX7d$cFNdgbD`e){F`9(zd+17LGDR&UR(m zqcp5F3WSnT)z8>6v2-jGHt;(YR|yRBR9CfaIdPTR8s50>qnor*H(9{kDq*9CMGj-W zB=}e;mk7TG&w@l47)Qb{LN<)PGEY>#G*f%;OuMtbJhBYDbG7j@T-NeLQl?MlGn?n_H6m(cI= zdn%o(nqO6W2EwWSM|TXmAf)$cwwC#|0j(=Lf+%5mr$%c4y}hvJ1XelrFP0SN@1}?2 zhI7TUZM@Zjy2}seP7DR+Px!dp6Cn*f;*kwhKkwWsy}(K4Shdp^$?Ep!HsIp%FWS5t z+|jX21(Cf$*PlU#pXYF*c$ujd2|FA7a@9PC+g*oEqr2WDALB8Mrg-OcCZC^e`IEa`Xk1l@!XYY zGwS$x;Yct62w;_`bED*yTnn~=3 z^t;o#xaTe3l4wmiPsWmxk{~PcG#igz#y7$C7A4HH=w*knyM(M4Lw;4swDz!}NO-IWc;9a{FnmYZv>FJgmaWtc)HZGj6{ zpfJ&zmLwcqaofknyi?}KT-(S6XGFO^J`!+B7*rtLAhw7ZTti-aYsHqr>Pz zd{g^y6;a4sn6rq=P}5O1XGHKUSYtKe_#rd2G_tV7>7v#EHE8recHyh;8Y;H%{j&(M z-^*8+Th$tz-WvF%PF@s>{wEKP6rsY9#9Cn59s%tn))5zp6Et&vs4u+(;6l?NA)|^2 zXHUy=hoBn)lmM-}Q!UQ%5}h?(8(f$Cr{Lo}(GGHRFU3?00 zjWQ}LF2x{MeOV1P@Ya5^BBo)!-xQZ2s0XClDKro#Fm=@M@TzM!_EE_^bkR(7T{ShH zHm`A$*j_6FcHLt>IAzb;`4*vou&p? zx}A9#!U5+&K5OdIjX1_|l^&lJnQVzPfvntDoh%dxV^QQb^bB8FC1RO`xf8741S!(St(_3p{&{|1aMESllXJS z!r(f;v}?}t_XYc@Mq=yl(Q~Xbhf{ivSJIu3Pk|2@-NV^OD0-$}5fAI@AXd7j;GHIW zEBTCBlKt49n)@>8z~3;g3z&myiEEv2yq+hIGNh21Uq-|8P$d;sDw+AmxBBHJeK14f zLL%V9-QaW*$w4vx$Rn9#)Sb;52DGf^oARi?9a-t*j7&W_ z`BNzKlmswNymUDV48flhmTm+-;NohW{0)TPY}k`|vR3A;tqwB0mgG{91+sN28mX#H zF!F~C@bEc<8{y&nFofpUXll9JzU?Yq0uCE?T`|GLP#k_dj)CbRs~%e&Vr zUIeO}sTL0$O8JlD^2_Z=O727%54*HxQ(~)Nvz$~GDJx|{G5e?`acB`Q@Dj`LNS!j0 zV=EhlZ48I6f3w!<4gXZUa~lfzEbN4?EnO><>%9$gsS9c^dV7zttu zK0>2$xgD`ZNnJ`+yg0o@QMUVCoV75{h6SKSIO%w+keT|VUw_iiqsWG)A5`XZdYkUc z0;HS{I1f&wc6WA|9As;evqVYVFi9jo!6knfsPd>Z^}HN<_w@2VLkfnvYe>0Ut7J|l z;qZDM#R{y@=KHHQ#&2}j)XA_$Z@w|PXsdG#bZT#Q>=xTKF_5VRG({jY_@N(d6?%hv zM^Z+cm1`bK?*tu4M(VQikF=vDZ6~`Q8&~<2==EY+4XZbKh|edgE^GdlNZpmHR z#+L6Pmd}l@mYs>*?C;#8Q?bdL1YA`sRJrS&nB+>jFkeVAXN~t$8jPUdA-zu13IH`5 z{*(rG>y8{aM^0TM{wnDUlSd-MF&>w-GKMgsr%n}#VFshM6=&;fhhtlzPvAL8LK0CP z*eGVrwFgVP$(qNi_V=aa?ejZ(PB1Q#XzzwS(<(?%vOS!3bRhXdc#O=R7T(x27;^Zk z;{KcL+s=pg=J%5U=Q4npVTX?9aDTlwz~II8yTGL5dZao5w{_E>X~!9PWAD|DLk#!o z&s`g4wlCk!v54_&v`RgYv+V65VH*6$JY2zaZw`!9iB9aOVOGftdf}dCfPZpTK}q|r zKOi>D(8GJ?abX~mnPyzdozgb~Rk@cge6o}wqWG;o$om+a8nwa%J}gY5 zstR{so<>oj7mN&>^Cv%fr(Dk%GnBg4A=lLO2to;url@I?_3>EWVyP7@%wnvPkkhlt zDwru#E;LXu(dkb2OC=)Nkzb&SZNw|PBiY41sv;=Ksb)<}w5Y|t%6|!&nfAsKtFKGY zk*exoWq@D}V)NlDD7|>bTkGA3Nh?evl@}L6X7S5^f7i7jZm&( zAQ>Z=QsO{MeXRb0=UL;RG^p2WMlkGX5-z{w%a*FKks@cDaO2;;M!Urv(@^!|+%Q=Y|M zG-o|&zKiskH^;F4#XmYG^SH^^md-Ad z&ugYMP{XX3*z3Qx=MtN!kBm`#I(tNxnmHSTtjrCnh3e>zDqOM;riaR1PDdZLwdx!x z`Xa(=`8}!oDjLD1v@LII)ASw+p%+H~7&`AMp9S!L@pt1FI#M4NfN6lc%Iv5a*vi zTj<|}uyLk565e(r<%Ns-J$`flM;T?1M&Z|$pbYTragBz%%g4Eg$FCnz{#C0%SFoEP z)VvTn^KmCF>HU##K#uE8qCr%;_86XB`a@V1Wcm!;R8QSUj83wp_ z)xp1zt~oR;dQ`agrM0lnq;YdC+E`6f-2Mqxz^gJ?GG4TV*JpIvdZ`-EFD^(4@nL5H zMM?f1-R?FXd{rlq4N-wC|Ep~7JeSo*ki1F_={lO#F1zfkUO=H2{WHW`fD~R-<9;Bg zB>%)E0-j`sJiMn5$1Xtbf=^I-%}WOxME_$XOXN0EHT{YQWaj3h)7ha>9C~Y`py1nd zxPZL8i;&@oKSs2$jzNu$PrtbjM!hI4V=;wIzcR;A``K)7l3wTXxs=z8GTxot+a09U z4QxyfA44>cJ6B?Qdxl~24n2I^*xI6%pUDUdHRnfwY6DL@zMhCTgmB+V6xrO+UqFn14G#&JQ0wk4ARIvI@9e6|-3sT?;aA{c?Bc zu@}bn(M-c#MFDdz?oU6!4p*DHsvJ*vmnnYr$B(2NBqVwcgYal*4EWoIHw(?5V$9wT z4s&PzIqK(~dhz1cNsPDlcJ?6lUq1Y7tG0pe8fO%?#rq*OgZ)nav7cU$$DrHmmGi1o z7;bR@ID-icpCcy(uIfHx4pT~yB-sj$6Wj||HtBnKyQ&U7MCR_>A$kn0wXHhb*!bUG zVeJT?r;ejM%HtDh1Zy-_(&F#(ti714qCQo8@;EffyD;q-DHsz2P(9G{$>mKz9C76F zyk@9bAJlm&zs#Uige8T9F&dv({)%+})uHS}x+nM}n3_)lUPKg&ji4~XdrOgP_W@F) z`LNE5S`nxXYkKL)Qj6^Qwd48>p~lL}S9dD;IP~YGXJ+vBmM!SvAsQEPJc1AGiBJ(N zF)>BrA)ZDd-a!p&1_wHbGf^R~+XHZ=#1nOb2WDpGJFK-MYu|!Y#b;)v-fjqtSA6Qi zHh167R)Q;?A~%D<4;CKIWzYFnpNI{}8)3_T_w|qbjgA^#Zee@al&N;3kAG>)jZ8Kf z&mb;i>GxJNn(uf_C0y<4G6j_TE0zUotjK4fdI*F=LiwK(08n^#6F4qfWEhBEO>15- zgx2_J;Q*aM??i3bJPY5S_p`{M_C_+yc(W*$TeGo1Ju#hNOBhu+U6Avi_nTg7P(Fw! z6FHB=e=KDz%M>J!W2^e?Kv7CCTtKCVplZ5*sw>O{73db-vKF~y9W>G)l*!`y9Bugt zeMRX`QYf3rCOe~b=K6q#)~zuG4ZPSbMOtHiHp!KcAiA3DyLRJ|B+ZACvP-@VvTRlu zYhHtwBTukB`xwzZ$x*w8#j)o>`4pw6uAC8P^dW0N-2FS(B7T@I>lt~1!sFOUQ|f{M zvcPLEHy(=WV^X~& zD-Q8D#ZWn8pDqM5v;L63f294wZj-o2&Mzo^4~wJ{;QvMW<9D5eLVBIVlevXMumaW% zA+tFKg(lM@=f@dsqQE%3(Ob>E74N5lAq|gp^)&~0lFdrfxgQc9JFo=F_iGhsO{Dlo z%!=HPc<7200|zF&??~YQSbJjID>t2Z=iZv&#mZT?rLp1rJBWW*3u3SqX^>)7P8^CHlwhas^VHM-7g0gY zu9st_T&OXTSZ!Sa3Q)zRvg9`am5IHA53`gw&VA zNjCec4AeSnwY+-cn4f3$Us`|55SrrSp`ud(oX@c4dBTWd# ztqq%x*?=Bl%-oQeC8Nk~tWpPE>KwykS`Df`PBt;aOK~j^U624$WB=+nOdO-J87>iB z2G2@%%{ zt4U=H7`s|^*vjnY+Qvq-_3_p;4}bd~Q>^x{nh}bLjQzTTA8uzu(Gj)PxP{AjXpB`P$6*4Lih03Vh7eGP)M`RoG% z0FPxW+UDBGLM-93FFdj ze6(PRTymFg8J9a~TzgqXR5gKd$Up?tPUC^2pbCo>>pUBmgJZ`tLn@4+C@$EHen#?) zu$qqz9NIJI$?-9(yx2~Eu3DzeP?dvlZf30=Kct|xhtqUibMAp%;7`?}5rKXi)gzh_K$x}y#v>2H!xHS`Tc>Bh+G`e^eq zS8^^X|KD!JZBc1y61WkBa_x@=t@%@Bmn^(e#& z!5tPNK5cta=77f zV?T*?p2@RxLTEA0Wb+9B8D=Wr1l|r9fJRt%$Bj@Hr}#WPccTE9Xf)K@1`b^>ikhbq z*c;s=E;#vw{1{_y1P|n53&5|44-KsiyYuR* ztK2k$ZI%HoZPRg#1D&T70Y|6erJ^oN7~taYyAde-Jw+x9I#wurxqZVb_ziO<-LK;x ztJJm#xN|!_7~X0nEFM^Mbh^rCf$f?z&nPW5lu)$c`hOdA7Oxi-c7iLQr3g(z9aaq1 z3&Q;M#-1ZP2lLdt=fMV?R;OxO;na@@K4$+ut?LS^zFp=!>^?|X5AW{tM$p+64yGuC zaF&F$e*Qb7?9RmPUtZl;NyQ3WI@zs#`nSWWcpR+tmM~9tibS6_R?d~j7e@(4b1p)( zK9s{CTs&Bn_SuF{S_1SbL0k&8*5z<)K0+jEQR*P#p>ME1UZ0uif&^BmYZmA{5+dnY zT-q9ur+zu-js{;SwI)Mk1uqD4`Q5*D#MB36 z=Dn8UzRxzuuvul^c(9-52#@$=ba2UGV6W0O+Nx>)vLBHTSeeRpI&P2}RtNXUb~>f4 zLToNZ)kfK#LnKAPj1E@A!C~ns8(B+4z|E>r^#6DDhC|`e6G!U{JVWA5}(Qeq4j7LVvOBp z4?%x6x?OMjU6F;F`;1Nq0cWG@ef*aW<7O$F-Xr)Oh?QQ&)na%uH^a;F2L_ z2?UqzhN^zR{&w}YIws_E@lQR&#vh2oxAtYsn(TA}THi~}B4Qu7s5z9q;nTwnln}`0 zn8&d-oqB*HW*AHVr|R{rhxRAKaV1ez12tzYwzvI0**_+$kc?RcQ*R@m3&$Q zl4kEfMoWuA1#;Zag#ZpsUS-p>XoR>pw%?h1<|k0qmr`%)q5<q(km1bvkwmU~nzEL;CVdl9m6P8Sft#t+6FxE6KhHBWXo?LpCGN@>}CuYa3wUhd@es2>x zI!^z7_uR9$u$!JsAkLEgeFY=-SoYRm#OM?hOm3+W>)3p}tgq)vO1K|mOqEQCshQ$2 zUAjablrLIoyhBxUm)l5I{15>`0t2H`REhnH4URpsz&peuCbxeP3Qvwtz4i~pWN?#s z(W!y&o{SkxUVivQ%5KgPhO!-1^C|PZFnT0)nRrbnjao4s%hhoeO&95I-t55T3ujj_ zpQX=(QZMiy(c^ZN%3Pn40*C3X#zmpEXjPr-;7kqW4IPyB5GJ_%*utpFxl4vSmsO6? zKMD9cDKI0%?Qw}t@miTaG_jki>!~jKJoIw23-4a7(hnf**rc8h-)=cOa=dH_^?=fA z_cf847<<-OVHD?s1-fMFZTT#)`1+)>{r5KUm;4YGqOa*nnMbg_vcnW0KB5G-e?;X~ zKS*>Q=dUDc(l0zDnVVKqpV!G?ZgyPPN4w|(2>i=2=$E3RUNV6Fj1Ri&RVD;2!M#yh z4a=V`u+l7Tgy_?e9)5pLdS@s++#}MEnvEq`7D?=>+E`1EaF?Aap^rPaDSn*PKo-%? zRtJ~e=iL#VRYA9jYib-3*KTE2q{-x9Z=z-K%FR@G(;?!D*OI=dQ1zTx9*pc*h~tFK znU`P{5~%!4tm`0jW}Dd0-qd(H9VinCr~&**>EDRv11EI5_E9CQ@KtW9%@`JQLNkLF zfwVahGngtqQ)q^& zd5%@tMM^@RItGiIj+?fKrgIzKF`q1-2}MjQYl6k$^rJ*485*g&+VnZ+?-?03>m{d&H)GLW|XrbFF`S=Lbqv)1G$P1EW%6 z4%qo1ue9%u@b860@5AMepu&w(DP#Sp6649E5sB+q%Ad;P!4I~|c^Hj|tpvB!K0(MX z4ib=9R6Xr_4xJ2sqgTk~934%s`!R+<7WH@L>8>McGS7nB-<`jV76+DRlw}nK7LV`l z39%GO@0!|8=s<5RK*gG_c*%JE&796Z_yUJWC;qA#AJhC<9&b|v*%Ju`Kju_%_X$=U zl)gsswuvh4F&#ztS`+M#)$7bwf98?-U$-S$EX<}6P_vZ!H)05pcpd2Y{xxSEGfTe`(yNqa z?N56U>IS|mHJmcfHx~>Z_ae%ohFQfeuq|AeEt*>ev_jUHL~hT%=N(($WUid+F#{Pu zC;ff@*E}3=FUC~0^pEVGe3DQ9L0EKrho5oW>kP2nH#Di$PpUp3eBeAzDGPA-ktSt8$V^suy6&*NJ z{h2?ymc_Qbm~MUJhL4r`>@+qsosjYrqw;v=T_njpG~Wor3%z2w7^>{UPG_R!*CPzE zi56ZM>M23OcNlobByT-hG(CE|({#}k!|XkmHvM-@TJ2tpyV+L+>MV_nwaW~@(o3WW z{UC#-^uMAwUa*24-$`j!$@>qWRw^yk)%OFuuTdaKUe@rkX#Nn6BYI2n&Odn@;5=wX z9^qThmAud}!zRe|zP2A4fb3uNyqr2;h(4`p`Tps-r0#FpKCf(`_Dv-x?B_($CWLb) zNtJ=FA#XT7dzxA1$D6H7nt_Xb5`@8c3m<+xA&)A}*yEMcvZsvmQy44%R*h=(4 zsV`$rLctx>uM5GdWc6&~0c~~1$lGp-H&56zWLrB}B`mqoDT4exagWLgxdrh~1w*(x zvx8do2E)IV&_Rh95`~}zohf) zx%ZRiTqCF8(MG51AHWM>&`-OviDjLZSh_}GK#uNWAXIGJv?%F@Kypu$@q?MV)S{{b z;f*fr?N^)hDU_AXe9li6TPRjYs;+3Dy3G5R=X<*_TYvYM;BsV<`OI`5s+l4C1ex+L zUG>`~z@Dc6`-6ls=<)dO;X#{oO-a6eHVzu@Rzs{?M^|l!zKxc5ADpBIcrb~m7ayoLm=@n`&)0KU%*YTe2#}lYKdBX= zkJOx%etS7e3V=5DUh>L8N>}JkY1HF>FUTLo?b8V7mlJi4u zj5mdRXGOc)e)UCfX~rQYlv`ez;rB&n3c*epz5ZS7^{QXv_-Uc zlVnhersb3LEjI)r+p_KpX6$XV=`9|=9cg_}8zmJ61FjH9XA8TUUQYojlQQ1ZCn)&R zT~eIXLcP?_I{JtPzisUnG1BqC)M}r=K(>Rr-Fhu4YcW1p8VL98FPSAc%lW~NA*Wgq zwyEL}aIYAFxZI=a9o3@}S0AfHsBzaxOT#R<#R}Od4I~L?z}`tbkKtkPUw5V`{|t-7 z>t+uMFupChsha>7ewxF7M+5*E<#_TQb>Dw*Px==PY1oVZ%n{?A7u+)xlz`*qveJfM z-nznoI3+d1nB5P0-tb=x534L6R*};WxKU+`W%2u}Wi~Lzxws50?f=>vduMaado1ba z=g1t~^1|Aj=cEYj?OLO668U=AhbVROqESrWM2*S`n?Ry@bmj%n)siuoksHJnEH(DL zdswn7RIix6oMBgL-#vm~j+^P9=Yo{vpEHBv|Kj1RqoV5GaEA^7r5Q>Yx&?+tq-Ln0 zyL0Fqx>Qmc2N+-|1%}R{LqJqw=x&sf5)t?i6i_ez?z(rav(7qyowLr~`+48zdF$~5 zLIwby(%my50aLugL-KclJQHu+jt|XB3WI|$K0AmX9^Qui&Xv}oPqWo?Rr_*N9RkhU zF}Nx#So=|97xFvjrm%}WXZ6Hm_p4c;uGE zt?g~T>)dR@1h$t5J6x8f5p33{%tgeqo)qq6u5~Ek+NL-ZrNB^tQv6J$k>OM#{iJ+C5F9olgu8X~w=7IN6KQ;ZhGi>-7wOsw3C-WvH)!p*g zc=0H++%tSPz*Fn9>uzpfMQ*qwW}~0SQny~-x3e>vCMUP=D^A2yhsP6UT`I0A1+$6} zcfMt$oOl}s@9tEOtz?C5(N<7PSfnH`FA5|j3p2U;@%oRB%Er%FuM@IEX znEQH85mUP?AL4iiAnM=Mbrp4^BqjRG1{;OEX?o5!)XS&0XHtx;T596yhoh0hf;}xv z38t_+!cPAJ@G;>IvJ^r}|7#L@AtXddK>QK!A$4H9Hvrvg4fhRbyw@2~D45kz@hpYM z=E>8HH7_&m=b*wu=lc5(>GuPK&JpYT>}x?Zzr6j1f&2&;jAyz)$4b6qM2)90%Sn=E zRYd*t;;hF6qsXO;B{HY`^T>{fWu!PySHrk}iowEib@%K8A0L}EDXs0sQHlu2^KUQL zj?(`OS!HR|=E@^tK?ccJgcsf4M2L7Jk8`=DD~!r%a}2>_nY*)_gDsd0Mv>>6rW0oZ zF8*nVK!-FO`4$_~cAyHEQ&O|K+#w}L3C4u_O|psxhT5&w84V1m3K8gLoevXxZN}nn zb;}BrtCafH=ppq;+6jrL2r0mXUtF9sNU&Re6o*sELUF)*7B90<1Y| z4q5*)JCvQZoYLVOUioPSp|8uGKM;24^3C^5n4ebp4e5w2vymsQ*Rwg7@im@J zJJXV5fADP-wk~lsS1giN4qshKjN0E8C$Lul8D?Q;`I$D!api0~%jVdQ9ZhRH?AE9K z+NHfPETl~b5vG$ksm^#LY3=2h8{j6tJS!(^ZHzqp!`#uadG7yoZ29sj)v7SMVfjy^ zrg&nKGENEbkEs%o00;n&0Qm2SHQwjt0Z54-;n4};5ip?*PPduR7?KdZR>lY!-F1;u zfiJYYm1lneF1h8m;s}$GB;1hMN?P~^Q}|WCW}Ik%X1M1-O#%v=qi!WGl%JfEX5I5@ zZZUsT%QU~tO(5pn2upMnVq3~GM~p7#=QVK&JlA?H*u^lY9`B^Xvv3k0#E=jA{*3uv z+hj$9JnJvSH=EI76Z#W`b8(lHC)0oD=bDmaJ#fAgm`MS|%X{azW>wvXX@RwKvHCb< zJAFvdyge>dGI>2u0Ic!Bfn$1>jYvaUX@fIM(Pu_LP};DpHB6S9-#KW6Ps@%MDnxlH zAePQE)2Ep8OAfoG)h?~c(W2`XSk0ndnfgq}CvKNskiyL2H($etMPXB=Vws;H^PSB( zlO!y5LV^EMJcsUv&AW^rOd|jO?Y)EiQH>(aw0xD1S*?b{HYtY8Ea1Fh?Ms)_6(ep& z8n3xXE6&iCVxd(b$}DE*!JL@9kA`?+eg;BhUXwm)ERS)URMzM7e$IMEf#Kq5=rNke z9AoVM;WwBiYwknrftz*{U-^5HIo*eEKHoTVVfz(h@>a2hy)feZ zZL!)f#gsFq=jETB_FY>i9Ea4|DeaF^bB3DEcISlcf4dj=D+_0^$vv@6(6(V}$Xlu< zw&Y|VX*K0fHN~U&Cl>+OM1bSn0Rr&9c)^hfz<>V^fd7Jp@23L=Y?h#f!r4T+KgojNVSV)dRWOh zr&$I1$LjtC@Xxix1odY-C0z&heX0KW9F%u0J7LCN0X8vndsv~kt%a0EfZ7kY!=?q%p^@@-yO_dvQQZKoifnEGd5Ne<69i6){pcP#6K1W3FSVL z4B(g=NNqL~+Iwvjp;q1DG?FW%R}&XeNTu_gAMwzP&geX4yI8j?$naC?ox4~lOR%x5 zfAtqIl@s?`FnW6`t436)*&Xq_hbpw9UqXg+QA?IJYokED(R=>Y%#>B`3Zt*{DSN?APQSu*p&$A}=gcfJA`pE~mQBgCwM=2Ew#T8F6 z+iSSqUqqSNg}lw@Q#>MuEA(G8q-@wtzSjNXoRH86Dwta-R+*CQ-9Y_;c9ef}@ezi7 z+6|AJD9QPX+jbn?KJ2kcUk9ds)%osgDl+D7FqR&i9xtdYSkRDAM{}t@3b#9UU4f_9 z`ED4TKQRv6&y-97a`?F4rhDyxmOQND+fv1DW>x;Dj%HJVo-4-Ep&neEoVxvb?+ zh4r?uiHclesg>C#ZiY|CqmNA>YHnF6?*3gJkeDaBLFq>NCL#OFMmajXO5mizA3D5Svse9WvVQ^h{Zd`izHb0` z#`vB{tlShIy(|lu_}cCK#eWx7=RvRO1>5@Em9CKa@nS5tK~mB)mMRthVA1?)dDPz% zOFxqw$rkzryYF}$?BvYW| z$*Ys#dgcJzwu0*6jLNZ#ADUcyhoCEvv*JH?Ou6?15*<4WoM{ zRIlPc1#EJ%&%uwfImd}4tuFNxL3hprIY?z?%ripp`E~z?d82YY+@3BKO|@T?v_biB zz|fHxukU(+9V1O)5qB?JuavIv$t1t3_F9}Rf1kDQMlg4ZNbx72SwIVGJ-eQsymn4_ zD1ABhrMjUR zq=pL8)+J@OvF~f=PKR(|?p~%T3hUUs5F2w|qEPFrwS2=bPfMx_Q^iXZ`v2n({x$9r z-boWj0N>97?h`PNHyI*iGi^u4usxiqxKI;4R{77nJ7q4AsD5mCI3O}_bLKddpNg89 z`1ZNIK-;NEAZH^b=cck-;-&BvJPCC<>)o3Fg|<5VAPHv@*P$^cXPoXbTG*!b5p~BN ziNk)Gs7Os5B@HvLSB#8WJp?5A22)d7XUh3;7L)I;$z2D%t7Yq4Fqk1=DswvyXh6CX zsH!--CELAwK~Camwiyvk*|%r(jB=wz7IK-WPPgUVv4+FtXx<62xYJR)5cVNae48|_ ziFBsz{|4&B+AfP8Y9q>$E=A@BsdbLGofK#deZ*RWe#_Do?zDf#2D|T#3M0@Db*CJ_ zpBtp%GLFfHemc_cc_tLfn=#?f6YPIPStq|`c21Y(8JtRwPB!635SSwUuPi`JWyZU<@i zoNzCFsFpe7;edqc$m6&Wxi_ghw4ra(9vvLSs>?x}$iK!)TIV?h`_hztb&?GMtBnPc z+&AnMNw+kJUp)RcneHxV3<+x<%%nwV_I#SDfT5f9tY23jubl)BQPW{RF3uTz;FTR5 z(pt(w1%?S92KUY!$(Y$UY$VL-315VDbOPT>B)5^MV|$+xGs7d>_LB1vfkjf))p~Aa zv$;m3Ug)9gu#WKZlh|A6FJy4O!UoJ{c78HU%jh__SvpGCbgjVgFe`}r8fk5}BW+lp z{8II3fkf}aR&k-;$4q7=`Bx(F2tQdwRR&a_kRm8>r!cIwiGZyaWq;q>-FZbJBl z|1(ia8@k>O+U7nJMg|>suc|9KS?T0uznl1sw6WAEjTE*GiR!j5>=i_zvyo*Z10LG@ z0W0qh#VGPI_kC23bx@fAtK0Od7ypW}A2E^I|C<&90Q{@&1P~6rdaTItD1wr})H^d{ zJwrus8aY6Jq+Y_7^|J-|-(zP82qT9vYSDaamfCcd!IT3+Ob1bAMCs z*6zM~NrYXB&B_HX#i5)!9(FZ9j+Oprr}69|L?A=qN6Zjzd!&+XrnQbNB?pt1lq(-+ zqlOGmC;8U3nr;(zM#5e=$e; zKL1h}2eNKV`$N`rse;hpZ!$k7Ltr^g@Tx3ynk#-%@?7(w)i}A*3~8WAgmlJgGVy_q zu4)!zX+;KnZJke|$xdQq{}XIM4|}J-tr`>_dF=0&Q+G7yto_j0B)|_`x3I&j{V134 zC*y3TTm6zlTI}I;lvstKlS|gcoH6;sfl{4Ien?u9ev8IO|1?HrGLGp3fcjG_wkYt< z7dDKwZQQ?jJvc> z!&IOXa#qMmzPjClx8``T8S^bkFfcIkw)wormTwjTPF}b6N`4tUk`i$o6QLHm(Bu3i zGT!A<#rkov@Q99EIcDt(!GFmE8g!eJH+;ZcF}KrG=8&>G~B71CA$ zRccKDm+333>{s^=hr7AAQ;t38?leI!F7(ODg32rj@AUIahl_{5(MuRUDoC{GUjV$$ zWcgs0&e*9UT4_n+fFhfuB26Irjkw#S#-heVG(S~L0zR{&lahDpQS?~qVKfv=hEF`2 z>XMpY;hD7@E1*$BXs84&ioZBsjoW@PxWm2OQl?Q=cE4QtVxo#KRx)F`96y1MTUqJx z3xa3>t!P^|?}tafL_czr)!@z%mHpU*KVr;B1?L%Z+Fi4x!BO`F+>V_t4 z>3=fK2>sPVTPKUtts9AXJ;TFETh%N~z2mt|vKS~|yfXzb;8OwY?|9W600H<3|7O|J zp|Z6%%eKupcx0h@=+X(P!%zFRT!mk8xVf?l>1SERdLj3ZFzr^iS(3)@X}o5BX1GOuL=KwS z5?-MK|CszQruYJ5s8;%lt~WWEhM&$*SyjVkF;dQM5UChmRukvnODO=N56{9 z4K2P>Kp>^5W5GpA40=O!OB6$3?W!x0!@?%BYq^&Cacn(TM z6htbe(b*6pdX-WlWr)7M;TwTC0}x!y0G}2p6#MShM>1b@`P`0@eQGCVfjD&5ij7xS zHrV~z4BNnKD4szyFdw5{MgqS$pF!CL2*G^aMV+2n{rpkVeJ5xM+aQYjTF~{3ri4dt z@Du(wuLHnzw+E_tFaBW%pKFvyDKgmDb$Ln1Ecfpnw8`Fl?xL(-f);o!4kTlF07z!U?tyO9#eSaoqu=;ZW4LkCJ_)&?9$bznQe^ zNt$0RDsMP2aLa^vq^Vg^*S%nObHAX1d?M$PTi8%_4+2s9Oj}{^ojBnk?=XO|C|0Co z?Qj%yd@TTMi`HRSS zJdi-V)dN+IdQ@8se}>1(B{-_8x$><^GuIK z=DMIF(;OKM8X0Wp%nPqD(1=VDD}7U|fXG#&mUhBpzA@2O>5G zTly5(ZFws}o?VA3W_s9m=H-9!_A~}Eaih&IL|zdD<2OC116aqdD-kv)=2FD&{{^t4_ERG{ z$2CrFy+2#n4%3;G`43%cBCNJ(O+de*4Hlxmb!2(nZXnWqifv!de?5{OW=J*7plG1q zV9(GIHebSi8CcB8P{jm(e%Vzu5$q>OpF~dqu>Zfqy59lp3I6RBJp}=_FpV$ik)ox> zlt5LP=@rOh?NsB+Rr(1J3#jBM%XW+IRY|+qX7z~$i{3Zcf-aUxE2iP#<#g+%PjL%e zPU~}=5`8)f`@C+TPMa05JN;2V9?P~F&nZH}@OVeVYCu>|l{aBi%js0d^_rByDCYt)S9{4b(%2eiW-yYU$z+FBMc9g-wXh zk{14EY(eCLZ*b3@WhpLo&_{X^*t1A$UJiA>Ydqo_XU7@ru zXH__dYAQUQgb8eao0+4{y4Us8sfMCzhAQ4I#36gepSN>vozn4mRwUJNf|h*6$E`y= z1)TmxX`oinW4&ORNLAa>XP7~GYF?%2Quc?z2AW^F*wfj3RYEfLc*n596G&X-UzCjr zfI^W-gXwa6mvYtS6s(059)N&8o=`J;h4w_PG4IyWY;kP){XOK5*`-D2XIp9WX}A9T z0{KpQj?ID^dA=LFHj(=OB)$0q0Kq@=_wj$gqXGcM`I5vla=)L#UZ2RyZB7c~dX8P5T=CQO z;$q(pE|hTpnTT`cpZT%^cD>(P*E{%LlKq+XZ?tp7E8}0zpw4=cdwPL!C=Sssyrnd5 z2IHEf1e{rPAL$(b>7+}CoJ+6h`o2Ne8ktFMXc>#tXfNrzGB4d7n+b;T{n$Cq(r*?F za$=Wqph+*EnK=Tm`X5_LN5Rd!x)mP87fFD zi?S>Hzc!dJ!gra2zEZkM&tKiIBJpzZ2C_#YSU&LS$Y+V#ge>o-t3u=n%1r79YX{P? zUkRLbyTx-Eycl-llC#JIWC@-M(;prBDRe_bzn;uyP+w|ju2|>faVGT`3xO8|Be-Up#6iL(eCly z43@}bc%>S%-HT>gw&-96on>=+M;~0X^;^L1IOe?}x6gy^x*0WtJI=0~$(5Ebk;%6^ zcCRJJpfGgvqR?y9{QcWs{EAi*p(_X4G#uLkvYbM%wvIh1mRgp(98cA-c-_TANc1p0 zpWj58pmL?Nc!AJM^1z7_5LFw;k?ee$V_lM}cQ>>t-gjL;o*82bG0zvN)3dL0r7hs{ zn8a}cYai|Dra4IPzwe}TW)2#HRI|0E1|dp zHB*8Tg(?6q@eK&l%tbP0lIG1}yCvN5_?lPJC`sJ5)mFWR`WadNhWN0;_X^pET)2(< zynK3r?mKm#h5U>Me1<9aMsZ^fAKW`OjXr78KRl+~S)R3X9g|4kYu@03qH%^U_r2BC zI{JCU?fig{ZaW?$<3BpI8{EqlIXCo7=^}YAs*XQ!Hc*-%=DfIGwDfj+kM2 zj5qC;#V@9RklZ>38)>Dlx#O2`3!>%YlzWeQDAkq=tH==&*>PX&k@~Z~nX`n+YN{GD zAOnp%GTHQ|t~E9-fBMD1yO#tNbvZGzZ~}g%V?T}$BW4s?Dk)_R=5mGtnaw0YXvg@lO94Rc$_sgv|uMLwjib%E*v2`#Tq>GsjHcczTG>EpWS zq2CfMWvq25+Z?IBoZ8J*d8wY3f_r}fR?r!5oSkFytjMeH?~3222naR}LjLhm<~uWu zkM{!apIAs@7{9}Df8I>34zRy-QC{%z^^@!;`lYVn23pxwXulcjfWx?tKY8r?=Jfd0 zdqiofR#-XBqvD}KM0_7#Z8hjD<&gr3r+$h zy~YIE++uT%;P4FggM?X^%ui2CFae3aNP2n_L;2GljsMd7id_(%%qJ;esF(}(@lmS@ z86~xDOv0-JuqItRgiBe&5yAx_V554Ga0u*x?;5*Y;#C;BgvVD^9l$UFLy^qoTHc%Ro+L7vf4BI^o4>O!;9yP|&Yj%zeB*=z$F>z6HD1g-$c`Piz zh9O9G>*mnqyiOvtp_n=v47p$PV%khCN=vVG@!F2qG`>RDN}2wb5e~Yw(bqqtDKwoWgJ=8YXLm@BUu2YCmK^=DiS*! zQ(XNJyo8;}K)EVB-My6aZ#29V)>t|JLm}?9?RATWTdBP&Wx*I18OUsaWrg3j<(wHp zJ1q)LcG#t1$rq1D=j6otb#Kr8w^5pG2AgUP=wRB_4`vm4yPa3y(Od!*4L9ODC;aiB z@4dq-06;p$iE$g3c=F;`&H9X=)=jdN7yQB%Zrch~KN7B=JBPyi56pK)_KM%X?A&WW z^B08C<1oSTG`TT14_r3X>Z~fACk$=B>=+xEmsdO|Q44#vuONjxPy2SLR!cu#H3nLq z@#RX)|8@-V*LEGKvso#;p-)=Vm=mfhy+3wRBMo5^v zH0%M59@=Qb8%hfEuv$j5X~kqBTlMW0pELVf&Jldf*q?@I=a3Hs zEk~`A=)z<;uYJ&8z^maunxhOxx{c>7Cl9UnG@CCv28dsq{31dB0?Cwf`kof~3{AoM znxb5yx2pBED>JRKT()@^pyXe`eYzn(3qvYahJU^;hN|Vfuj5l2F#jPM$~S14DpQ+PDkco!0;kUNw2SLk_^NfHEDi}f_)BK=Y5Bth zV_#Q~V7CejYLwaU@5tupmX=;xu_FY*pI~V>nO*AUTb3VAy%i(D^bosY=E>>U=a{a3E>VA?yB(YBMEz&w5{bVrd8pSAS=&b540jmZ5kUnoda zZm^)D%+kM@7FG17zZE%44=q?Zg&8G`)cssOI^52>k^@((4-C&~HT&GKA%pEzW7=#k zrPA4|5;YZCd)PD$o7$T~gEuF(b^NJn!c`eo7thVTgfdx$@3F3;o!rOqrU(~mh}9_0Jj=}*rK)%8N`Lz zH*DfDo<8J_`fC~~9klVz!SMMh2ul!VRqres0ba{KB2En}2w9%HPXlT#UN|+iIY_ECfOmvuoj?$~R)69o4Sz|# zUWJQ}8hi8&rE=A?44@Z2y$eXJ7Vye~#d=9pq-v#9Xkm|3Ow)Emd5Ous!M>)*F1a1R zW+X^kGFzv-v^E9N(Y?CB4kQP%P z?06b_=QfB6^Ddvyj&^2&hh^Ve7BmgVho51Rby)A!u6S7<+x0gbrA^q5iSE$jr=P!UDU6ki6p5YDK6R3Xl9O)8%s!nN@>=zYPa%vzQ zfKZCT@ff`xs1;61AEQ73zy^ZVHmLIa5dvFIf z=;QPpkrrSvb91^E8vkld2>_&o`1Y_*!>{MNU7x9A_k-^HHS%Eu*qEa3LwLQCyvE$n z!gUR-$aIsg%#~x$tZ;GbHSm864Md`?&r#%&T!fWyX&vCOr+{8e>CP{71#fHWT$Pk9 z?_r>F-Rly@`#ZFrJ~o2rs`y7Ae8r;}pphP%E}V z1#UrKXHf^7V(?_3p3XB6gX29?+{Sw_~`C|b~D4J zqVsW$jN= z7fW6L0yuy@DABj#ja;TdeoDo-pG23pgwKZDSws!7>h2s4AtQUWfpAMn!%Jzyw>^LM z7!PcSSXt1mR_miv5xdgAh`WB#ES!Cg{ z{X!4i&(0Zk1^HvF_UeI|d?=mi)-SYy`?W==Rd^r8RyTa>ru_Sc zh?NPY4C9c4c$B!=zhU%PXp;5z>})+YedR3|GdgRSXvYPS#$gE!A$S!hr6MuyJ3*aO>9;x%e%ozLy zq^pg&-_Z39xzi^dr-=(fnTjmsxZhILl=d=EN0caE2bO5gDcdaPH~JFs&!fkZ-Mzf> ze`}33QfdFD%4->4s_+bGp>jSFb^VQEGiA_%6N+Ujk~0C!DNGq2;c2EObt?t9qIkl( zxO?Q~(CopQR`^1q?+OTy_pa%|CpfR3iV?55iF|HccDrP*;!uF;EF{42#u3(&jZ%(G zigTq>O(Ffg4TtQU*z#Hc3xnM-7rSng z2@VSNSUpAz(lgS`^6(%z4Zt$p@K)_KYAdIrf+2Nsltd|Qv-x~OXN!4Dj<{56YU`(= zA_JI!_GvCxIL@FaAnawY}O~9ZcnBNY zu)s0kWn2nl9~abkf;>>Z=2S0W*HdRWO_4C=6j@<11nropH29*g!O3wR6IYxjt|i8V^?TOlH%03JNfEFck)&*i$o%vtc+TaU6uy1PYN;@Ipp zS08EQiuVldDtCb7W#;h1HgfG(65DQwy=dVpmQN5BJA{Ch25Rcv{)jrvW~m#bNIvwN zn)@4nB-*Ly9Cj!l(u7* z5M|}_+kJdn*vq_r%fkv6Ic(#hTdw!b>B8|?Q zY^BxNpBMItmfOS3+1U*}L6$!RHEBZufjr>%-Jl7&^w5zu5LMh5Y~N5a^~Hvdp+ssL zXq32)?+vM?s;26uKwx&M*c+M{_RM4mO)0aM5;d4k4nR+qijVo%x!cU4nKud1Qdjg! z=5ME#MO0Nxy}s414=XRdKl}7!_*JGJyL3lz*w+9|dYH4O)p|&<+_HZfk9!=LM~Z^b z_G#Ax_oVO_)jnz_7A;T(3wlo<-D9e=;g!NCS#sAQe!n$pCmtFyJSB3m2CLbCA-d*1 zOMzJYjyRIL6bxe7x20sE-*9)z&Sflm6k!st| z2Z4)#&YC7^x>aKeYh}V7QbXUu&3|;Z6i`{BBr4=@R_YJ_$YzjWzzQ8kd;HjNXE#ueqeuhV`)MY-uMs_BTwQKa; zIdly&FPda|I>a-j^4DLo@a@-+MsPsPcm1R8MREJA&=|Y(LUr=PnqfWnKo=!_?{Sx? zarJMsCz9{U@Lun0Je6m@Ny?cc4LS;z6;#a`7np(O*z zT3fP_H9~J{#>HcZpT#fk6>Mx?K%cowWNty0O8&q+j}*Z2h%~FD18TvmHNq+MHDKUY zud)4X=ef|@+0&2U$_$ntVW*+qYaKn(sR3gk0X}2v=dnJHRlaWN0#{_mF-@#F+pJxr z&}D}2n>d!u6)6T5@ys$8*DU8mOy~O1;f8YIv}xeh+U$#ikw)fcz--g6yEF11mtwho z{sI--9R8H>)&0GKxe_1kRHh|4a#@r0aztq?JT-~;TDovq(6r;}aJP$9D6HL0?)uIa z&(ykb1rgp{{lXlOtfd3FmCm0+XxeaVZ1w_Q$3(ZRRpOo)crD zLw^B+PxLxvL48HELf&V%M!v1kfY+?kE6KW_2FqQZ7dRD$H$tS;9n1G|3N3n4ny;OK zQpsfhsp6LFxMY>Bh`b2>x%i{Ty~A;@n;;#5avND}+b<)sg}jHkujJr;vYe=I;K{0! z#oJPo_+%5wsP|g^Dx9r>cT#u~=;Iz4P5`w3@k^&{z#His*9@s%P!GoH7FXxh}(Q>DeVf}P-0(}PZN)3U&1 z4OEvqb%}~wH#rEK4J*`bPyK>l#;4o=F2ZTvQvp&rUBrqo;-XiuO zu$p6jU}_VL-6`v-jXmj&esRq1 zjzPdqsCi`Tug4QtrEjcyz_DFM;zI|s;<{G&(w@;;!vEc54O-YCcKPCJCOCx;aNgXz ztsQ+I%@=j|oKWF)pBg+rKxIv+mlz<*t8U>@T;;HO_oW0gypAp=cm$(u7ms82Xw`9Y zD_76G-e69T9-#12r*KR%fEMmy=#C=lCh7<2qM6qcj&La}s3%B%V_oDIoOrO-zI|4y zQS;kY&Gl)$FfP%F6PjnOrmD(o{!Nj1G`6MimVP3fky;-Up>!%seZkTjNXr-&2dobqen;5u9M9HXu{)CT4evimk-=O=qm4Y_tU#EBiK z?K_VeQnM{KACXJ7VOgf$&4In?)z}vPGZ|b zsV+UwuiFznjW)G!w&S$mqda-(D!rhMjPf9;wGeR?U9#1{%~uWfkafV$F$KO(L$~T@ zHqsO#pR=fX8Vcu9Z8&j0D_yMTBU-u$AKkSR0N8uNGg{0pCTEvY06J})y4|6%$d_y= zfxFBVm++<|W*d#n?AtX)X;cMaXYaTq>nE!I(YNm09z4Ifbi!%*f0uCoX(cva0H^&0 z2)TV=_*TR?)xO0>7L~C)Y5f=Q8;yatQq!*5RWrjjG&xyc2Ng_&x63wTV@V1bTx zHVG=6>YEZGp7#lO8uIb_mjZ7OeWs6_s0BDK43L-L@OwB5+?9azcFm7m%@)8)Ma~ zW6c#GKR=U`fA(}fb+q!C4p(iU(9z~=gy3slj;F>#Ve~sN{qT}oX6Moq^2DKP$=#}O zs}!0&%{b4O-69d&$QKp;SgxVbcAPKLdF@fGGd7(CoszHPc zGO59c(?`v4Q&kDv7^_j!$Ti4FZ`D+D^Gy3?0Er-~z$rM9t_Qfc#jKa5@Y#ca?_Byy z3_|}aIhw6-=;JOqGRvzMAH}1PhJpXkcRNY8PT{R((oDkld%=_cI!a5cvUb1*$+(pzwyG!DfQW}v;Y1m?}S>XVb zIZxZpUTDym_ra7;oj^5?#}{h*bm%TWUvm1(RBH#*tVU4w+P%rrLv;pz+&!!aRIVu7 z-5+Tn?`pA@H*~g$>UiVkosmWV$rwWgHkmFUPo|XjGYO?f8EMe0`xZqb==a++Tll{{ z3<$HdI(3ll>O4OWoE*yQNt_#(r41Y!6w!zLQHo`@jf2^KUnY+%WgqVf21Q0^FKEnO zs_mMoCY=2gAQ{0NX*lNE%bh*Zq^2ytP9bbpZL|`ft7+gbd1}hj^XC0`#?I}&-5aF! zR4CXr>98Q>#N)D`SKizBtpsc_(**e!fRCFfZPr?M?sRUekq2zGB4-yW0#r}bwVq`P z7T5U`wm%zS!yMmpI+be|F`)00WhqOiO7E11TBSX}2hvp;n-C~L$Vv}%@Uqley>@Oh zw~2*`m^-sDq>x7?7{I??yqJ7w5)INFFbhuR2F;#7n58m5x@58Id@#N@mW4s=7hM|| zq`MylcFIn4?DOW2#?Wty@}u|=sHc8ro$v0K znvt)ujpvaPXyG+SJV{3?u8;L7O5vd*vSUF4&z&1#9n9=;K(()4tf%f%Lep27BL$Ih zHadf+t_*Wt0YT@nhK(k2r_6OJ{jao^1JU7@A=o}f#N+yjvvAmA)3=3@@%gWrZ9ER^ z?@aUFM~h)CjAJ`L((mu03TGo)!cGImocwopOXcHGj}x|}+~4J4z<7)gn-WjQlJ{_0 zb@%oNwL`OuN_gFo7C5gwY>!AiaM=AeAFiWc{-N|U;ge~`=xaOdTT~lo0l}S3IkF`) z6(#HyTGT3MT**^*_h#ugeHsX_0GjB|ObBC^oSL6umf8xtOb@VpT3e@lJtjN;+mR*O zbJr+l^7bGh%}hD@i^+J*Y|M)>9pWr+&@_F1k|zBd4+=a1evToq zr4lt{0skJU3|+Yf>wfd|-A8V2mkp&4%`d-?jG8ki>d@I#Rwvn1k;x~fefC`6qx1jd zI(_AfoyA91{Be{H4ExehDzEzX&B47ooiR!*{)=l%eU@HN{B`ia$2Tj}Zy6r^$GSX0 zx@9}J#IKB?LSfa?_gBqWHYFbu{gD~r^=b94Q`d6Sq@SAqD!zXG_=PjcKlX&*T-mnL z@^<>2FYA|dMy&tRSu1nQ_@(sp@}jc8FMm9~cRqG)PMluu6w|rWH|PsqI^HOJ-*$KE z?D@BPkF0Jf$hxwku)LCuN$i3}v%uAmqfwg^TR0}weKt7fadppDyYvfdKR!RN&2}>P z-j?PAM_voe4fap?dui#~)LlM5clsIrPmtQ7wlhMZsr%$*=f29b2RHlY^_6&UPuM=` zChO{^GdG`RKB#_amhAK`ZK_`V_Ki_nU7qn)>k4+9e>Ew;)PU*j%$@SFd*A$X+*0}= zWvQcodCdgXX2FoQy^~+H1UIHF;8A*~mTE8U%x<*)>AGi^^s4R`?>Zxum;Z70nWb8P zvbx`}Zdf165^y}~%iL+dcP$I$Z9UJ#={nW%!sQ!m+KZE0UhXTNvQuz-rR)9(=JyM| zFU-F*>DqP9m20m$ukKapdbWJ>)Vq>5zFc@&@&6RpiFeN>ShoC$`Lwq0NA$$4pA|ZM z7aurtBj&&Ab>E4*1KGk7JZi2=F}~i!>Xv!7Z~fN&5vf~~PHNmX;qTnKRAjYjOVlBe z>1({#^~@}*T9*<2h`HQJ?%1sDA^{~QEO{?jYp6O{{5!it;a9ymV^QPuQ*BF^osjf( zP?J_&!FA;Lnf_~ACn*PHtaN|2pJ&?{j%VRAD`#2o74WV9aMmPm`rnq<5y!c>Os^{! zzPPz*($)v_H!()-w)?Lx#Q64~<+UV`N!36Jeay#C0}e~ zomau$M<+eM>FEW&>9zWmth#rQiP218M1!yp+;t$|K7<(wj8$N?mZYZWRb>`{ Z&Al@Ni&G(C%fi6I;06qeW352#3;+t#2@U`N literal 0 HcmV?d00001 diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index a54bac0c4a..b31b9edec6 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -22,6 +22,9 @@ import { ZipArchiveResponse } from './ZipArchiveResponse'; const archiveData = fs.readFileSync( resolvePath(__filename, '../../__fixtures__/mock-main.zip'), ); +const archiveDataCorrupted = fs.readFileSync( + resolvePath(__filename, '../../__fixtures__/mock-corrupted.zip'), +); const archiveDataWithExtraDir = fs.readFileSync( resolvePath(__filename, '../../__fixtures__/mock-with-extra-root-dir.zip'), ); @@ -31,6 +34,7 @@ describe('ZipArchiveResponse', () => { mockFs({ '/test-archive.zip': archiveData, '/test-archive-with-extra-root-dir.zip': archiveDataWithExtraDir, + '/test-archive-corrupted.zip': archiveDataCorrupted, '/tmp': mockFs.directory(), }); }); @@ -152,4 +156,15 @@ describe('ZipArchiveResponse', () => { fs.pathExists(resolvePath(dir, 'docs/index.md')), ).resolves.toBe(false); }); + + it('should throw on invalid archive', async () => { + const stream = fs.createReadStream('/test-archive-corrupted.zip'); + + const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const filesPromise = res.files(); + + await expect(filesPromise).rejects.toThrow( + 'Timed out while unzipping File: docs/corrupted.zip', + ); + }); }); From cfa078e25554e55d89df97a4f912d1d3c89106a0 Mon Sep 17 00:00:00 2001 From: Otto Sichert Date: Thu, 16 Jun 2022 15:08:41 +0200 Subject: [PATCH 07/20] Added changeset Signed-off-by: Otto Sichert --- .changeset/nine-mails-crash.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .changeset/nine-mails-crash.md diff --git a/.changeset/nine-mails-crash.md b/.changeset/nine-mails-crash.md new file mode 100644 index 0000000000..3dc4334f17 --- /dev/null +++ b/.changeset/nine-mails-crash.md @@ -0,0 +1,9 @@ +--- +'@backstage/backend-common': patch +--- + +The `ZipArchiveResponse` now correctly handles corrupt ZIP archives. + +Before this change, certain corrupt ZIP archives either cause the inflater to throw (as expected), or will hang the parser indefinitely. + +This change introduces a default timeout of 3000ms before throwing an error message when trying to unzip an archive. From c878f1c9f34467a9f029362d85f7aad1f44f6648 Mon Sep 17 00:00:00 2001 From: Otto Sichert Date: Wed, 22 Jun 2022 10:26:22 +0200 Subject: [PATCH 08/20] Fix linter errors Signed-off-by: Otto Sichert --- packages/backend-common/src/reading/tree/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-common/src/reading/tree/util.ts b/packages/backend-common/src/reading/tree/util.ts index 1d7a42c93e..49c696467c 100644 --- a/packages/backend-common/src/reading/tree/util.ts +++ b/packages/backend-common/src/reading/tree/util.ts @@ -44,7 +44,7 @@ export async function streamToTimeoutPromise( }, options.timeoutMs); }); - await new Promise(function (resolve, reject) { + await new Promise((resolve, reject) => { stream.on('finish', resolve); stream.on('error', reject); }); From 6cd69466745392c4e1865917019f8cd6d9a5b5f6 Mon Sep 17 00:00:00 2001 From: Otto Sichert Date: Wed, 22 Jun 2022 11:17:40 +0200 Subject: [PATCH 09/20] Ensure test ZIP file can't be read by Node 14 either Signed-off-by: Otto Sichert --- .../reading/__fixtures__/mock-corrupted.zip | Bin 34594 -> 34649 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/packages/backend-common/src/reading/__fixtures__/mock-corrupted.zip b/packages/backend-common/src/reading/__fixtures__/mock-corrupted.zip index 51c4c268c7c1f7b0ea240777eb602af2fffe826e..5130a72f6fc1c51de7cb1a9450777b9d4d0938dc 100644 GIT binary patch delta 65 zcmZ3~$8@ugX@gR`xk6$=L26z~W?s5Naei)UNd|~jlCMybk*WX`2Qib1QWLX*0-41M NQJw(`X_+~xTmUe+7l;4= delta 9 QcmccF$F!)AX@gQb02Q4B^#A|> From 2b59c8ed4dda3879ab2ccd75e878a9c0a4dfd950 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 7 Jul 2022 11:55:40 +0200 Subject: [PATCH 10/20] chore: made some progress moving over to a new library for this Signed-off-by: blam --- packages/backend-common/package.json | 4 +- .../src/reading/tree/ZipArchiveResponse.ts | 135 ++++++++++-------- .../fixtures/test-v1beta3/template.yaml | 96 ++++++++++--- yarn.lock | 9 +- 4 files changed, 164 insertions(+), 80 deletions(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index cb2f653277..40beb3deca 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -41,6 +41,7 @@ "@backstage/integration": "^1.2.2-next.3", "@backstage/types": "^1.0.0", "@google-cloud/storage": "^6.0.0", + "@keyv/redis": "^2.2.3", "@manypkg/get-packages": "^1.1.3", "@octokit/rest": "^19.0.3", "@types/cors": "^2.8.6", @@ -48,6 +49,7 @@ "@types/express": "^4.17.6", "@types/luxon": "^2.0.4", "@types/webpack-env": "^1.15.2", + "@types/yauzl": "^2.10.0", "archiver": "^5.0.2", "aws-sdk": "^2.840.0", "base64-stream": "^1.0.0", @@ -64,7 +66,6 @@ "jose": "^4.6.0", "keyv": "^4.0.3", "keyv-memcache": "^1.2.5", - "@keyv/redis": "^2.2.3", "knex": "^2.0.0", "lodash": "^4.17.21", "logform": "^2.3.2", @@ -80,6 +81,7 @@ "tar": "^6.1.2", "unzipper": "^0.10.11", "winston": "^3.2.1", + "yauzl": "^2.10.0", "yn": "^4.0.0" }, "peerDependencies": { diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 501113d9f1..6a8e5ee621 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -15,25 +15,27 @@ */ import archiver from 'archiver'; +import unzipper2, { Entry } from 'yauzl'; import fs from 'fs-extra'; import platformPath from 'path'; import { Readable } from 'stream'; -import unzipper, { Entry } from 'unzipper'; +// import unzipper, { Entry } from 'unzipper'; import { ReadTreeResponse, ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; import { streamToTimeoutPromise } from './util'; +import { zip } from 'lodash'; -const guardCorruptZipStream = (stream: Readable) => - streamToTimeoutPromise(stream, { - eventName: 'entry', - timeoutMs: 3000, - getError: (entry: Entry) => - new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`), +const streamToBuffer = async (stream: Readable): Promise => { + const buffers: Buffer[] = []; + return new Promise((resolve, reject) => { + stream.on('data', (data: Buffer) => buffers.push(data)); + stream.on('error', reject); + stream.on('end', () => resolve(Buffer.concat(buffers))); }); - +}; /** * Wraps a zip archive stream into a tree response reader. */ @@ -76,15 +78,13 @@ export class ZipArchiveResponse implements ReadTreeResponse { private shouldBeIncluded(entry: Entry): boolean { if (this.subPath) { - if (!entry.path.startsWith(this.subPath)) { + if (!entry.fileName.startsWith(this.subPath)) { return false; } } if (this.filter) { - return this.filter(this.getInnerPath(entry.path), { - size: - (entry.vars as { uncompressedSize?: number }).uncompressedSize ?? - entry.vars.compressedSize, + return this.filter(this.getInnerPath(entry.fileName), { + size: entry.uncompressedSize, }); } return true; @@ -92,29 +92,36 @@ export class ZipArchiveResponse implements ReadTreeResponse { async files(): Promise { this.onlyOnce(); - const files = Array(); - const parseStream = this.stream - .pipe(unzipper.Parse()) - .on('entry', (entry: Entry) => { - if (entry.type === 'Directory') { - entry.resume(); + const buffer = await streamToBuffer(this.stream); + return await new Promise((resolve, reject) => { + unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => { + if (err) { + reject(err); return; } + zipfile.on('entry', async (entry: Entry) => { + // If it's not a directory, and it's included, then grab the contents of the file from the buffer + if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { + files.push({ + path: this.getInnerPath(entry.fileName), + content: () => + new Promise((cResolve, cReject) => { + zipfile.openReadStream(entry, async (cError, readStream) => { + if (cError) { + return cReject(cError); + } + return cResolve(await streamToBuffer(readStream)); + }); + }), + }); + } + }); - if (this.shouldBeIncluded(entry)) { - files.push({ - path: this.getInnerPath(entry.path), - content: () => entry.buffer(), - }); - } else { - entry.autodrain(); - } + zipfile.once('end', () => resolve(files)); }); - await guardCorruptZipStream(parseStream); - - return files; + }); } async archive(): Promise { @@ -124,17 +131,32 @@ export class ZipArchiveResponse implements ReadTreeResponse { return this.stream; } + const buffer = await streamToBuffer(this.stream); const archive = archiver('zip'); - const parseStream = this.stream - .pipe(unzipper.Parse()) - .on('entry', (entry: Entry) => { - if (entry.type === 'File' && this.shouldBeIncluded(entry)) { - archive.append(entry, { name: this.getInnerPath(entry.path) }); - } else { - entry.autodrain(); + + await new Promise((resolve, reject) => { + unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => { + if (err) { + reject(err); + return; } + zipfile.on('entry', async (entry: Entry) => { + // If it's not a directory, and it's included, then grab the contents of the file from the buffer + if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { + zipfile.openReadStream(entry, async (err2, readStream) => { + if (err2) { + reject(err2); + return; + } + archive.append(await streamToBuffer(readStream), { + name: this.getInnerPath(entry.fileName), + }); + }); + } + }); + zipfile.once('end', () => resolve()); }); - await guardCorruptZipStream(parseStream); + }); archive.finalize(); @@ -143,29 +165,28 @@ export class ZipArchiveResponse implements ReadTreeResponse { async dir(options?: ReadTreeResponseDirOptions): Promise { this.onlyOnce(); - const dir = options?.targetDir ?? (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); - const parseStream = this.stream - .pipe(unzipper.Parse()) - .on('entry', async (entry: Entry) => { - // Ignore directory entries since we handle that with the file entries - // as a zip can have files with directories without directory entries - if (entry.type === 'File' && this.shouldBeIncluded(entry)) { - const entryPath = this.getInnerPath(entry.path); - const dirname = platformPath.dirname(entryPath); - if (dirname) { - await fs.mkdirp(platformPath.join(dir, dirname)); + return new Promise((resolve, reject) => { + const parseStream = this.stream + .pipe(unzipper.Parse()) + .on('entry', async (entry: Entry) => { + // Ignore directory entries since we handle that with the file entries + // as a zip can have files with directories without directory entries + if (entry.type === 'File' && this.shouldBeIncluded(entry)) { + const entryPath = this.getInnerPath(entry.path); + const dirname = platformPath.dirname(entryPath); + if (dirname) { + await fs.mkdirp(platformPath.join(dir, dirname)); + } + entry.pipe(fs.createWriteStream(platformPath.join(dir, entryPath))); + } else { + entry.autodrain(); } - entry.pipe(fs.createWriteStream(platformPath.join(dir, entryPath))); - } else { - entry.autodrain(); - } - }); - await guardCorruptZipStream(parseStream); - - return dir; + }) + .on('finish', () => resolve(dir)); + }); } } diff --git a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml index 10d4a39ff7..c56e5a0d79 100644 --- a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml +++ b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml @@ -8,28 +8,82 @@ spec: type: website owner: team-a parameters: - - name: Enter some stuff - description: Enter some stuff + - name: Repositories + description: Some repo properties: - inputString: + appRepoUrl: type: string - title: string input test - inputObject: - type: object - title: object input test - description: a little nested thing never hurt anyone right? - properties: - first: - type: string - title: first - second: - type: number - title: second + title: First Repository + ui:field: RepoUrlPicker + serviceRepoUrl: + type: string + title: First Repository + ui:field: RepoUrlPicker steps: - - id: debug - if: ${{ true === true }} - name: Debug - action: debug:log + # First get the app template folder, and template into ./app + - id: app_template + name: Fetch Skeleton + Template + action: fetch:template input: - message: ${{ parameters.inputString }} - extra: ${{ parameters.inputObject }} + url: ./skeleton + targetPath: ./app + copyWithoutRender: + - .github/* + values: + component_id: ${{ parameters.component_id }} + description: ${{ parameters.description }} + services_app_port: ${{ parameters.services_app_port }} + owner: ${{ parameters.owner }} + destination: ${{ parameters.appRepoUrl | parseRepoUrl }} + + # First then service into ./service + - id: app_service_config_template + name: Fetch App Servcie Config. Skeleton + Template + action: fetch:template + input: + url: https://github.com/my-org/helm-values-template + targetPath: ./service + copyWithoutRender: + - .github/* + values: + component_id: ${{ parameters.component_id }} + description: ${{ parameters.description }} + services_app_port: ${{ parameters.services_app_port }} + owner: ${{ parameters.owner }} + destination: ${{ parameters.serviceRepoUrl | parseRepoUrl }} + + # Publish the app + - id: publish_app + name: Publish App + action: publish:github + input: + sourcePath: ./app + allowedHosts: ['github.com'] + description: This is ${{ parameters.component_id }} + repoUrl: ${{ parameters.appRepoUrl }} + + # Publish the service + - id: publish_service + name: Publish Service + action: publish:github + input: + sourcePath: ./service + allowedHosts: ['github.com'] + description: This is ${{ parameters.component_id }} + repoUrl: ${{ parameters.serviceRepoUrl }} + + # Register the app + - id: register_app + name: Register Application + action: catalog:register + input: + repoContentsUrl: ${{ steps.publish_app.output.repoContentsUrl }} + catalogInfoPath: '/catalog-info.yaml' + + # Register the service + - id: register_service + name: Register Application + action: catalog:register + input: + repoContentsUrl: ${{ steps.publish_service.output.repoContentsUrl }} + catalogInfoPath: '/catalog-info.yaml' diff --git a/yarn.lock b/yarn.lock index cd607fcc68..9dfb3ea4ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7320,6 +7320,13 @@ resolved "https://registry.npmjs.org/@types/yarnpkg__lockfile/-/yarnpkg__lockfile-1.1.5.tgz#9639020e1fb65120a2f4387db8f1e8b63efdf229" integrity sha512-8NYnGOctzsI4W0ApsP/BIHD/LnxpJ6XaGf2AZmz4EyDYJMxtprN4279dLNI1CPZcwC9H18qYcaFv4bXi0wmokg== +"@types/yauzl@^2.10.0": + version "2.10.0" + resolved "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz#b3248295276cf8c6f153ebe6a9aba0c988cb2599" + integrity sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw== + dependencies: + "@types/node" "*" + "@types/yauzl@^2.9.1": version "2.9.2" resolved "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz#c48e5d56aff1444409e39fa164b0b4d4552a7b7a" @@ -26322,7 +26329,7 @@ yarn-lock-check@^1.0.5: yauzl@^2.10.0: version "2.10.0" resolved "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" - integrity sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk= + integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== dependencies: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" From 6a817fbd4a6a68412fae9c4e0f0d2e02fe282167 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 12 Jul 2022 17:58:25 +0200 Subject: [PATCH 11/20] chore: reset fixture Signed-off-by: blam Signed-off-by: blam --- .../fixtures/test-v1beta3/template.yaml | 96 ++++--------------- 1 file changed, 21 insertions(+), 75 deletions(-) diff --git a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml index c56e5a0d79..10d4a39ff7 100644 --- a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml +++ b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml @@ -8,82 +8,28 @@ spec: type: website owner: team-a parameters: - - name: Repositories - description: Some repo + - name: Enter some stuff + description: Enter some stuff properties: - appRepoUrl: + inputString: type: string - title: First Repository - ui:field: RepoUrlPicker - serviceRepoUrl: - type: string - title: First Repository - ui:field: RepoUrlPicker + title: string input test + inputObject: + type: object + title: object input test + description: a little nested thing never hurt anyone right? + properties: + first: + type: string + title: first + second: + type: number + title: second steps: - # First get the app template folder, and template into ./app - - id: app_template - name: Fetch Skeleton + Template - action: fetch:template + - id: debug + if: ${{ true === true }} + name: Debug + action: debug:log input: - url: ./skeleton - targetPath: ./app - copyWithoutRender: - - .github/* - values: - component_id: ${{ parameters.component_id }} - description: ${{ parameters.description }} - services_app_port: ${{ parameters.services_app_port }} - owner: ${{ parameters.owner }} - destination: ${{ parameters.appRepoUrl | parseRepoUrl }} - - # First then service into ./service - - id: app_service_config_template - name: Fetch App Servcie Config. Skeleton + Template - action: fetch:template - input: - url: https://github.com/my-org/helm-values-template - targetPath: ./service - copyWithoutRender: - - .github/* - values: - component_id: ${{ parameters.component_id }} - description: ${{ parameters.description }} - services_app_port: ${{ parameters.services_app_port }} - owner: ${{ parameters.owner }} - destination: ${{ parameters.serviceRepoUrl | parseRepoUrl }} - - # Publish the app - - id: publish_app - name: Publish App - action: publish:github - input: - sourcePath: ./app - allowedHosts: ['github.com'] - description: This is ${{ parameters.component_id }} - repoUrl: ${{ parameters.appRepoUrl }} - - # Publish the service - - id: publish_service - name: Publish Service - action: publish:github - input: - sourcePath: ./service - allowedHosts: ['github.com'] - description: This is ${{ parameters.component_id }} - repoUrl: ${{ parameters.serviceRepoUrl }} - - # Register the app - - id: register_app - name: Register Application - action: catalog:register - input: - repoContentsUrl: ${{ steps.publish_app.output.repoContentsUrl }} - catalogInfoPath: '/catalog-info.yaml' - - # Register the service - - id: register_service - name: Register Application - action: catalog:register - input: - repoContentsUrl: ${{ steps.publish_service.output.repoContentsUrl }} - catalogInfoPath: '/catalog-info.yaml' + message: ${{ parameters.inputString }} + extra: ${{ parameters.inputObject }} From a69b0cfc98605b2827dd965a5443880ad6f4f62e Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 12 Jul 2022 18:16:27 +0200 Subject: [PATCH 12/20] chore: fixing things and making them green for now, still a lot to do. Things don't resolve at the right time Signed-off-by: blam Signed-off-by: blam --- .../reading/tree/ZipArchiveResponse.test.ts | 5 ++- .../src/reading/tree/ZipArchiveResponse.ts | 41 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index b31b9edec6..b94e424251 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -120,6 +120,7 @@ describe('ZipArchiveResponse', () => { const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); const dir = await res.dir(); + await new Promise(resolve => setTimeout(resolve, 1000)); await expect( fs.readFile(resolvePath(dir, 'mkdocs.yml'), 'utf8'), ).resolves.toBe('site_name: Test\n'); @@ -133,7 +134,7 @@ describe('ZipArchiveResponse', () => { const res = new ZipArchiveResponse(stream, 'docs/', '/tmp', 'etag'); const dir = await res.dir(); - + await new Promise(resolve => setTimeout(resolve, 1000)); expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/); await expect( fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), @@ -164,7 +165,7 @@ describe('ZipArchiveResponse', () => { const filesPromise = res.files(); await expect(filesPromise).rejects.toThrow( - 'Timed out while unzipping File: docs/corrupted.zip', + 'invalid comment length. expected: 55. found: 0', ); }); }); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 6a8e5ee621..5141448d1d 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -25,8 +25,6 @@ import { ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; -import { streamToTimeoutPromise } from './util'; -import { zip } from 'lodash'; const streamToBuffer = async (stream: Readable): Promise => { const buffers: Buffer[] = []; @@ -169,24 +167,37 @@ export class ZipArchiveResponse implements ReadTreeResponse { options?.targetDir ?? (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); - return new Promise((resolve, reject) => { - const parseStream = this.stream - .pipe(unzipper.Parse()) - .on('entry', async (entry: Entry) => { - // Ignore directory entries since we handle that with the file entries - // as a zip can have files with directories without directory entries - if (entry.type === 'File' && this.shouldBeIncluded(entry)) { - const entryPath = this.getInnerPath(entry.path); + const buffer = await streamToBuffer(this.stream); + return await new Promise((resolve, reject) => { + unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => { + if (err) { + reject(err); + return; + } + zipfile.on('entry', async (entry: Entry) => { + // If it's not a directory, and it's included, then grab the contents of the file from the buffer + if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { + const entryPath = this.getInnerPath(entry.fileName); const dirname = platformPath.dirname(entryPath); + if (dirname) { await fs.mkdirp(platformPath.join(dir, dirname)); } - entry.pipe(fs.createWriteStream(platformPath.join(dir, entryPath))); - } else { - entry.autodrain(); + + zipfile.openReadStream(entry, async (err2, readStream) => { + if (err2) { + reject(err2); + return; + } + + readStream.pipe( + fs.createWriteStream(platformPath.join(dir, entryPath)), + ); + }); } - }) - .on('finish', () => resolve(dir)); + }); + zipfile.once('end', () => resolve(dir)); + }); }); } } From e5a38fcead3933e4680b5fab6917fdb2675b572e Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 14 Jul 2022 11:53:31 +0200 Subject: [PATCH 13/20] chore: refactoring some parts to make it actually run a little better Signed-off-by: blam --- .../src/reading/tree/ZipArchiveResponse.ts | 118 +++++++++++++----- 1 file changed, 88 insertions(+), 30 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 5141448d1d..02683bac1a 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -161,43 +161,101 @@ export class ZipArchiveResponse implements ReadTreeResponse { return archive; } + private async streamToTemporaryFile(stream: Readable): Promise { + const tmpDir = await fs.mkdtemp( + platformPath.join(this.workDir, 'backstage-tmp'), + ); + const tmpFile = platformPath.join(tmpDir, 'tmp.zip'); + + const writeStream = fs.createWriteStream(tmpFile); + + return new Promise((resolve, reject) => { + writeStream.on('error', reject); + writeStream.on('finish', () => resolve(tmpFile)); + stream.pipe(writeStream); + }); + } + + private forEveryZipEntry( + zip: string, + callback: (entry: Entry, content: Readable) => Promise, + ): Promise { + return new Promise((resolve, reject) => { + unzipper2.open(zip, { lazyEntries: true }, (err, zipfile) => { + if (err) { + reject(err); + return; + } + + if (!zipfile) { + reject(new Error('Zip file is empty')); + return; + } + + zipfile.on('entry', async (entry: Entry) => { + // If it's not a directory, and it's included, then grab the contents of the file from the buffer + if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { + zipfile.openReadStream(entry, async (openErr, readStream) => { + if (openErr) { + reject(openErr); + return; + } + + if (!readStream) { + reject(new Error('Zip file is empty')); + return; + } + + await callback(entry, readStream); + }); + } + zipfile.readEntry(); + }); + zipfile.once('end', () => resolve()); + zipfile.readEntry(); + }); + }); + } + async dir(options?: ReadTreeResponseDirOptions): Promise { this.onlyOnce(); const dir = options?.targetDir ?? (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); - const buffer = await streamToBuffer(this.stream); - return await new Promise((resolve, reject) => { - unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => { - if (err) { - reject(err); - return; - } - zipfile.on('entry', async (entry: Entry) => { - // If it's not a directory, and it's included, then grab the contents of the file from the buffer - if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { - const entryPath = this.getInnerPath(entry.fileName); - const dirname = platformPath.dirname(entryPath); + const tmpFile = await this.streamToTemporaryFile(this.stream); - if (dirname) { - await fs.mkdirp(platformPath.join(dir, dirname)); - } - - zipfile.openReadStream(entry, async (err2, readStream) => { - if (err2) { - reject(err2); - return; - } - - readStream.pipe( - fs.createWriteStream(platformPath.join(dir, entryPath)), - ); - }); - } - }); - zipfile.once('end', () => resolve(dir)); - }); + await this.forEveryZipEntry(tmpFile, async (entry, content) => { + console.log(entry); }); + // return await new Promise(async (resolve, reject) => { + // const zipFile = await this.openZipArchive(tmpFile); + // console.log('hjere'); + // zipFile.readEntry(); + // zipFile.on('entry', async (entry: Entry) => { + // console.log('entry'); + // // If it's not a directory, and it's included, then grab the contents of the file from the buffer + // if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { + // const entryPath = this.getInnerPath(entry.fileName); + // const dirname = platformPath.dirname(entryPath); + + // if (dirname) { + // await fs.mkdirp(platformPath.join(dir, dirname)); + // } + + // zipFile.openReadStream(entry, async (err2, readStream) => { + // if (err2) { + // reject(err2); + // return; + // } + + // readStream.pipe( + // fs.createWriteStream(platformPath.join(dir, entryPath)), + // ); + // }); + // } + // }); + // zipFile.once('end', () => resolve(dir)); + // }); } } From 7642a1423347382f387e81baf298ecca874fbf3f Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 14 Jul 2022 12:01:40 +0200 Subject: [PATCH 14/20] chore: looking so much better now Signed-off-by: blam --- .../src/reading/tree/ZipArchiveResponse.ts | 101 +++++------------- 1 file changed, 24 insertions(+), 77 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 02683bac1a..66c3bbf026 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -91,35 +91,16 @@ export class ZipArchiveResponse implements ReadTreeResponse { async files(): Promise { this.onlyOnce(); const files = Array(); + const tmpFile = await this.streamToTemporaryFile(this.stream); - const buffer = await streamToBuffer(this.stream); - return await new Promise((resolve, reject) => { - unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => { - if (err) { - reject(err); - return; - } - zipfile.on('entry', async (entry: Entry) => { - // If it's not a directory, and it's included, then grab the contents of the file from the buffer - if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { - files.push({ - path: this.getInnerPath(entry.fileName), - content: () => - new Promise((cResolve, cReject) => { - zipfile.openReadStream(entry, async (cError, readStream) => { - if (cError) { - return cReject(cError); - } - return cResolve(await streamToBuffer(readStream)); - }); - }), - }); - } - }); - - zipfile.once('end', () => resolve(files)); + await this.forEveryZipEntry(tmpFile, async (entry, content) => { + files.push({ + path: this.getInnerPath(entry.fileName), + content: async () => await streamToBuffer(content), }); }); + + return files; } async archive(): Promise { @@ -129,30 +110,12 @@ export class ZipArchiveResponse implements ReadTreeResponse { return this.stream; } - const buffer = await streamToBuffer(this.stream); const archive = archiver('zip'); + const tmpFile = await this.streamToTemporaryFile(this.stream); - await new Promise((resolve, reject) => { - unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => { - if (err) { - reject(err); - return; - } - zipfile.on('entry', async (entry: Entry) => { - // If it's not a directory, and it's included, then grab the contents of the file from the buffer - if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { - zipfile.openReadStream(entry, async (err2, readStream) => { - if (err2) { - reject(err2); - return; - } - archive.append(await streamToBuffer(readStream), { - name: this.getInnerPath(entry.fileName), - }); - }); - } - }); - zipfile.once('end', () => resolve()); + await this.forEveryZipEntry(tmpFile, async (entry, content) => { + archive.append(await streamToBuffer(content), { + name: this.getInnerPath(entry.fileName), }); }); @@ -226,36 +189,20 @@ export class ZipArchiveResponse implements ReadTreeResponse { const tmpFile = await this.streamToTemporaryFile(this.stream); await this.forEveryZipEntry(tmpFile, async (entry, content) => { - console.log(entry); + const entryPath = this.getInnerPath(entry.fileName); + const dirname = platformPath.dirname(entryPath); + + if (dirname) { + await fs.mkdirp(platformPath.join(dir, dirname)); + } + return new Promise(async (resolve, reject) => { + const file = fs.createWriteStream(platformPath.join(dir, entryPath)); + file.on('error', reject); + file.on('finish', resolve); + content.pipe(file); + }); }); - // return await new Promise(async (resolve, reject) => { - // const zipFile = await this.openZipArchive(tmpFile); - // console.log('hjere'); - // zipFile.readEntry(); - // zipFile.on('entry', async (entry: Entry) => { - // console.log('entry'); - // // If it's not a directory, and it's included, then grab the contents of the file from the buffer - // if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { - // const entryPath = this.getInnerPath(entry.fileName); - // const dirname = platformPath.dirname(entryPath); - // if (dirname) { - // await fs.mkdirp(platformPath.join(dir, dirname)); - // } - - // zipFile.openReadStream(entry, async (err2, readStream) => { - // if (err2) { - // reject(err2); - // return; - // } - - // readStream.pipe( - // fs.createWriteStream(platformPath.join(dir, entryPath)), - // ); - // }); - // } - // }); - // zipFile.once('end', () => resolve(dir)); - // }); + return dir; } } From 0f473ecdc7dc7ef9e3c0e690ae263fffa312faf4 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 14 Jul 2022 12:03:51 +0200 Subject: [PATCH 15/20] chore: remove the pauses Signed-off-by: blam --- .../backend-common/src/reading/tree/ZipArchiveResponse.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index b94e424251..c67dcf75d7 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -120,7 +120,6 @@ describe('ZipArchiveResponse', () => { const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); const dir = await res.dir(); - await new Promise(resolve => setTimeout(resolve, 1000)); await expect( fs.readFile(resolvePath(dir, 'mkdocs.yml'), 'utf8'), ).resolves.toBe('site_name: Test\n'); @@ -134,7 +133,6 @@ describe('ZipArchiveResponse', () => { const res = new ZipArchiveResponse(stream, 'docs/', '/tmp', 'etag'); const dir = await res.dir(); - await new Promise(resolve => setTimeout(resolve, 1000)); expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/); await expect( fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), From 773fc2c4a551e4b5162e8437e42e82481dd0b498 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 14 Jul 2022 12:05:56 +0200 Subject: [PATCH 16/20] chore: make better Signed-off-by: blam --- .../src/reading/tree/ZipArchiveResponse.ts | 14 ++------ .../backend-common/src/reading/tree/util.ts | 34 +++++-------------- 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 66c3bbf026..b40780d125 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -15,25 +15,17 @@ */ import archiver from 'archiver'; -import unzipper2, { Entry } from 'yauzl'; +import yauzl, { Entry } from 'yauzl'; import fs from 'fs-extra'; import platformPath from 'path'; import { Readable } from 'stream'; -// import unzipper, { Entry } from 'unzipper'; import { ReadTreeResponse, ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; +import { streamToBuffer } from './util'; -const streamToBuffer = async (stream: Readable): Promise => { - const buffers: Buffer[] = []; - return new Promise((resolve, reject) => { - stream.on('data', (data: Buffer) => buffers.push(data)); - stream.on('error', reject); - stream.on('end', () => resolve(Buffer.concat(buffers))); - }); -}; /** * Wraps a zip archive stream into a tree response reader. */ @@ -144,7 +136,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { callback: (entry: Entry, content: Readable) => Promise, ): Promise { return new Promise((resolve, reject) => { - unzipper2.open(zip, { lazyEntries: true }, (err, zipfile) => { + yauzl.open(zip, { lazyEntries: true }, (err, zipfile) => { if (err) { reject(err); return; diff --git a/packages/backend-common/src/reading/tree/util.ts b/packages/backend-common/src/reading/tree/util.ts index 49c696467c..bfe02ac6c8 100644 --- a/packages/backend-common/src/reading/tree/util.ts +++ b/packages/backend-common/src/reading/tree/util.ts @@ -18,36 +18,18 @@ // containing any character except `/` one or more times, and ending with a `/` // e.g. Will match `dirA/` in `dirA/dirB/file.ext` const directoryNameRegex = /^[^\/]+\//; - +import { Readable } from 'stream'; // Removes the first segment of a forward-slash-separated path export function stripFirstDirectoryFromPath(path: string): string { return path.replace(directoryNameRegex, ''); } -// Some corrupted ZIP files cause the zlib inflater to hang indefinitely. -// This is a workaround to bail on stuck streams after 3 seconds. -// Related: https://github.com/ZJONSSON/node-unzipper/issues/213 -export async function streamToTimeoutPromise( - stream: NodeJS.ReadableStream, - options: { - timeoutMs: number; - eventName: string; - getError: (data: T) => Error; - }, -) { - let lastEntryTimeout: NodeJS.Timeout | undefined; - stream.on(options.eventName, (data: T) => { - clearTimeout(lastEntryTimeout); - - lastEntryTimeout = setTimeout(() => { - stream.emit('error', options.getError(data)); - }, options.timeoutMs); - }); - - await new Promise((resolve, reject) => { - stream.on('finish', resolve); +// Concats the data into a buffer. +export const streamToBuffer = async (stream: Readable): Promise => { + const buffers: Buffer[] = []; + return new Promise((resolve, reject) => { + stream.on('data', (data: Buffer) => buffers.push(data)); stream.on('error', reject); + stream.on('end', () => resolve(Buffer.concat(buffers))); }); - - clearTimeout(lastEntryTimeout); -} +}; From e27ada575f0bae003b31035731948bd040321e9d Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 14 Jul 2022 13:02:49 +0200 Subject: [PATCH 17/20] chore: small re-factor, and update changeset Signed-off-by: blam --- .changeset/nine-mails-crash.md | 2 +- packages/backend-common/package.json | 4 +- .../src/reading/tree/ZipArchiveResponse.ts | 104 ++++++++---------- 3 files changed, 50 insertions(+), 60 deletions(-) diff --git a/.changeset/nine-mails-crash.md b/.changeset/nine-mails-crash.md index 3dc4334f17..ecb047802e 100644 --- a/.changeset/nine-mails-crash.md +++ b/.changeset/nine-mails-crash.md @@ -6,4 +6,4 @@ The `ZipArchiveResponse` now correctly handles corrupt ZIP archives. Before this change, certain corrupt ZIP archives either cause the inflater to throw (as expected), or will hang the parser indefinitely. -This change introduces a default timeout of 3000ms before throwing an error message when trying to unzip an archive. +By switching out the `zip` parsing library, we now write to a temporary directory, and load from disk which should ensure that the parsing of the `.zip` files are done correctly because `streaming` of `zip` paths is technically impossible without being able to parse the headers at the end of he file. diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 40beb3deca..3cc689f32a 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -49,7 +49,6 @@ "@types/express": "^4.17.6", "@types/luxon": "^2.0.4", "@types/webpack-env": "^1.15.2", - "@types/yauzl": "^2.10.0", "archiver": "^5.0.2", "aws-sdk": "^2.840.0", "base64-stream": "^1.0.0", @@ -79,7 +78,6 @@ "selfsigned": "^2.0.0", "stoppable": "^1.1.0", "tar": "^6.1.2", - "unzipper": "^0.10.11", "winston": "^3.2.1", "yauzl": "^2.10.0", "yn": "^4.0.0" @@ -108,7 +106,7 @@ "@types/stoppable": "^1.1.0", "@types/supertest": "^2.0.8", "@types/tar": "^6.1.1", - "@types/unzipper": "^0.10.3", + "@types/yauzl": "^2.10.0", "@types/webpack-env": "^1.15.2", "aws-sdk-mock": "^5.2.1", "better-sqlite3": "^7.5.0", diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index b40780d125..15bc0d0258 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -80,6 +80,54 @@ export class ZipArchiveResponse implements ReadTreeResponse { return true; } + private async streamToTemporaryFile(stream: Readable): Promise { + const tmpDir = await fs.mkdtemp( + platformPath.join(this.workDir, 'backstage-tmp'), + ); + const tmpFile = platformPath.join(tmpDir, 'tmp.zip'); + + const writeStream = fs.createWriteStream(tmpFile); + + return new Promise((resolve, reject) => { + writeStream.on('error', reject); + writeStream.on('finish', () => resolve(tmpFile)); + stream.pipe(writeStream); + }); + } + + private forEveryZipEntry( + zip: string, + callback: (entry: Entry, content: Readable) => Promise, + ): Promise { + return new Promise((resolve, reject) => { + yauzl.open(zip, { lazyEntries: true }, (err, zipfile) => { + if (err || !zipfile) { + reject(err || new Error(`Failed to open zip file ${zip}`)); + return; + } + + zipfile.on('entry', async (entry: Entry) => { + if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { + zipfile.openReadStream(entry, async (openErr, readStream) => { + if (openErr || !readStream) { + reject( + openErr || + new Error(`Failed to open zip entry ${entry.fileName}`), + ); + return; + } + + await callback(entry, readStream); + }); + } + zipfile.readEntry(); + }); + zipfile.once('end', () => resolve()); + zipfile.readEntry(); + }); + }); + } + async files(): Promise { this.onlyOnce(); const files = Array(); @@ -116,62 +164,6 @@ export class ZipArchiveResponse implements ReadTreeResponse { return archive; } - private async streamToTemporaryFile(stream: Readable): Promise { - const tmpDir = await fs.mkdtemp( - platformPath.join(this.workDir, 'backstage-tmp'), - ); - const tmpFile = platformPath.join(tmpDir, 'tmp.zip'); - - const writeStream = fs.createWriteStream(tmpFile); - - return new Promise((resolve, reject) => { - writeStream.on('error', reject); - writeStream.on('finish', () => resolve(tmpFile)); - stream.pipe(writeStream); - }); - } - - private forEveryZipEntry( - zip: string, - callback: (entry: Entry, content: Readable) => Promise, - ): Promise { - return new Promise((resolve, reject) => { - yauzl.open(zip, { lazyEntries: true }, (err, zipfile) => { - if (err) { - reject(err); - return; - } - - if (!zipfile) { - reject(new Error('Zip file is empty')); - return; - } - - zipfile.on('entry', async (entry: Entry) => { - // If it's not a directory, and it's included, then grab the contents of the file from the buffer - if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { - zipfile.openReadStream(entry, async (openErr, readStream) => { - if (openErr) { - reject(openErr); - return; - } - - if (!readStream) { - reject(new Error('Zip file is empty')); - return; - } - - await callback(entry, readStream); - }); - } - zipfile.readEntry(); - }); - zipfile.once('end', () => resolve()); - zipfile.readEntry(); - }); - }); - } - async dir(options?: ReadTreeResponseDirOptions): Promise { this.onlyOnce(); const dir = From 22e9262868b1bec6187645f4b50c62e802ed9bdd Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 21 Jul 2022 16:20:19 +0200 Subject: [PATCH 18/20] chore: code-review comments Signed-off-by: blam --- .../src/reading/tree/ZipArchiveResponse.ts | 27 +++++++++++++------ .../backend-common/src/reading/tree/util.ts | 22 +++++++++------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 15bc0d0258..8126d1fcdc 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -80,7 +80,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { return true; } - private async streamToTemporaryFile(stream: Readable): Promise { + private async streamToTemporaryFile( + stream: Readable, + ): Promise<{ fileName: string; cleanup: () => Promise }> { const tmpDir = await fs.mkdtemp( platformPath.join(this.workDir, 'backstage-tmp'), ); @@ -90,7 +92,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { return new Promise((resolve, reject) => { writeStream.on('error', reject); - writeStream.on('finish', () => resolve(tmpFile)); + writeStream.on('finish', () => + resolve({ fileName: tmpFile, cleanup: () => fs.remove(tmpFile) }), + ); stream.pipe(writeStream); }); } @@ -123,6 +127,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { zipfile.readEntry(); }); zipfile.once('end', () => resolve()); + zipfile.on('error', e => reject(e)); zipfile.readEntry(); }); }); @@ -131,15 +136,17 @@ export class ZipArchiveResponse implements ReadTreeResponse { async files(): Promise { this.onlyOnce(); const files = Array(); - const tmpFile = await this.streamToTemporaryFile(this.stream); + const temporary = await this.streamToTemporaryFile(this.stream); - await this.forEveryZipEntry(tmpFile, async (entry, content) => { + await this.forEveryZipEntry(temporary.fileName, async (entry, content) => { files.push({ path: this.getInnerPath(entry.fileName), content: async () => await streamToBuffer(content), }); }); + temporary.cleanup(); + return files; } @@ -151,9 +158,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { } const archive = archiver('zip'); - const tmpFile = await this.streamToTemporaryFile(this.stream); + const temporary = await this.streamToTemporaryFile(this.stream); - await this.forEveryZipEntry(tmpFile, async (entry, content) => { + await this.forEveryZipEntry(temporary.fileName, async (entry, content) => { archive.append(await streamToBuffer(content), { name: this.getInnerPath(entry.fileName), }); @@ -161,6 +168,8 @@ export class ZipArchiveResponse implements ReadTreeResponse { archive.finalize(); + temporary.cleanup(); + return archive; } @@ -170,9 +179,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { options?.targetDir ?? (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); - const tmpFile = await this.streamToTemporaryFile(this.stream); + const temporary = await this.streamToTemporaryFile(this.stream); - await this.forEveryZipEntry(tmpFile, async (entry, content) => { + await this.forEveryZipEntry(temporary.fileName, async (entry, content) => { const entryPath = this.getInnerPath(entry.fileName); const dirname = platformPath.dirname(entryPath); @@ -187,6 +196,8 @@ export class ZipArchiveResponse implements ReadTreeResponse { }); }); + temporary.cleanup(); + return dir; } } diff --git a/packages/backend-common/src/reading/tree/util.ts b/packages/backend-common/src/reading/tree/util.ts index bfe02ac6c8..63192102f4 100644 --- a/packages/backend-common/src/reading/tree/util.ts +++ b/packages/backend-common/src/reading/tree/util.ts @@ -14,22 +14,28 @@ * limitations under the License. */ +import { Readable, pipeline as pipelineCb } from 'stream'; +import { promisify } from 'util'; +import concatStream from 'concat-stream'; + +const pipeline = promisify(pipelineCb); + // Matches a directory name + one `/` at the start of any string, // containing any character except `/` one or more times, and ending with a `/` // e.g. Will match `dirA/` in `dirA/dirB/file.ext` const directoryNameRegex = /^[^\/]+\//; -import { Readable } from 'stream'; // Removes the first segment of a forward-slash-separated path export function stripFirstDirectoryFromPath(path: string): string { return path.replace(directoryNameRegex, ''); } -// Concats the data into a buffer. -export const streamToBuffer = async (stream: Readable): Promise => { - const buffers: Buffer[] = []; - return new Promise((resolve, reject) => { - stream.on('data', (data: Buffer) => buffers.push(data)); - stream.on('error', reject); - stream.on('end', () => resolve(Buffer.concat(buffers))); +// Collect the stream into a buffer and return +export const streamToBuffer = (stream: Readable): Promise => { + return new Promise(async (resolve, reject) => { + try { + await pipeline(stream, concatStream(resolve)); + } catch (ex) { + reject(ex); + } }); }; From 01d825ba28ebb63aecf7c1feb0c7f2275036a749 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 21 Jul 2022 16:31:02 +0200 Subject: [PATCH 19/20] chore: finish piping and stuff and making it lovely Signed-off-by: blam --- .../backend-common/src/reading/tree/ZipArchiveResponse.test.ts | 1 + packages/backend-common/src/reading/tree/ZipArchiveResponse.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index c67dcf75d7..f1915cf99b 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -59,6 +59,7 @@ describe('ZipArchiveResponse', () => { content: expect.any(Function), }, ]); + const contents = await Promise.all(files.map(f => f.content())); expect(contents.map(c => c.toString('utf8').trim())).toEqual([ 'site_name: Test', diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 8126d1fcdc..b25f6ad24e 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -190,8 +190,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { } return new Promise(async (resolve, reject) => { const file = fs.createWriteStream(platformPath.join(dir, entryPath)); - file.on('error', reject); file.on('finish', resolve); + + content.on('error', reject); content.pipe(file); }); }); From 764ba707a28f8f0737dceea151f31e9fed50a0f2 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 25 Jul 2022 13:53:54 +0200 Subject: [PATCH 20/20] chore: refactor and comment Signed-off-by: blam --- .changeset/nine-mails-crash.md | 2 +- packages/backend-common/src/reading/tree/ZipArchiveResponse.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.changeset/nine-mails-crash.md b/.changeset/nine-mails-crash.md index ecb047802e..ead5553d1c 100644 --- a/.changeset/nine-mails-crash.md +++ b/.changeset/nine-mails-crash.md @@ -6,4 +6,4 @@ The `ZipArchiveResponse` now correctly handles corrupt ZIP archives. Before this change, certain corrupt ZIP archives either cause the inflater to throw (as expected), or will hang the parser indefinitely. -By switching out the `zip` parsing library, we now write to a temporary directory, and load from disk which should ensure that the parsing of the `.zip` files are done correctly because `streaming` of `zip` paths is technically impossible without being able to parse the headers at the end of he file. +By switching out the `zip` parsing library, we now write to a temporary directory, and load from disk which should ensure that the parsing of the `.zip` files are done correctly because `streaming` of `zip` paths is technically impossible without being able to parse the headers at the end of the file. diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index b25f6ad24e..1a6658b0c2 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -111,7 +111,8 @@ export class ZipArchiveResponse implements ReadTreeResponse { } zipfile.on('entry', async (entry: Entry) => { - if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { + // Check that the file is not a directory, and that is matches the filter. + if (!entry.fileName.endsWith('/') && this.shouldBeIncluded(entry)) { zipfile.openReadStream(entry, async (openErr, readStream) => { if (openErr || !readStream) { reject(