From 2c972813414fe4b73f90ce412a9a5224e1094593 Mon Sep 17 00:00:00 2001 From: alindima Date: Mon, 19 Oct 2020 11:57:55 +0000 Subject: [PATCH 1/9] Docs: Replace I3 with M5D in specification Signed-off-by: alindima --- SPECIFICATION.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SPECIFICATION.md b/SPECIFICATION.md index 2985ae624ae..425c8b241f1 100644 --- a/SPECIFICATION.md +++ b/SPECIFICATION.md @@ -5,7 +5,7 @@ minimal-overhead execution of container and serverless workloads. These specifications are enforced by integration tests (that run for each PR and master branch merge). -On an I3.metal instance¹, with hyperthreading disabled and given host system +On an M5D.metal instance¹, with hyperthreading disabled and given host system resources are available (e.g., there are enough free CPU cycles, there is enough RAM, etc.), customers can rely on the following: @@ -56,8 +56,8 @@ enough RAM, etc.), customers can rely on the following: pipes are full will be lost. Any such events will be signaled through the `lost-logs` and `lost-metrics` counters. -¹ I3.metal instances: -[https://aws.amazon.com/ec2/instance-types/i3/](https://aws.amazon.com/ec2/instance-types/i3/) +¹ M5D.metal instances: +[https://aws.amazon.com/ec2/instance-types/m5/](https://aws.amazon.com/ec2/instance-types/m5/) ² CPU ms are actual ms of a user space thread's on-CPU runtime; useful for getting consistent measurements for some performance metrics. From cb9cc2536154ff52a4566035d0c16b63ff783983 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 16 Oct 2020 14:38:06 +0300 Subject: [PATCH 2/9] Update snapshot versioning docs. Signed-off-by: Andrei Sandu --- docs/images/version_graph.png | Bin 27391 -> 24903 bytes docs/snapshotting/versioning.md | 75 ++++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/docs/images/version_graph.png b/docs/images/version_graph.png index 57fdd9426d14570921292493f41df7b6b33132d7..8b636d3791d7f5d8c99f722212286bd3c8ef165f 100644 GIT binary patch literal 24903 zcmaI6by!s47CtH^9int9BGNIyFmyLVcQbU&kV8mGDWxDGjdX`}4j~{7(j}mDBPn^e z=bYcY_mBHr9%j#;y}xhmFIK$kUHiQTRDlqW8t>7gM}$g>vRaQGp)~^^5}e1t6;1^_ zqDPOg4n5@zJY9Tj9h_|*F@t3Od&bPiV-NT6WCqDH^YO_!+rT{C;jX|baNPxN?O^L* zWBuQId^~(SoV>ulpbjrDGf0Y05I6{cx%ha%M*nSZX=mg5KS646KL=-LOJ+WK0WKau z)N3_Mn1ib)+{2z3Bn|vla`m)v2EKvIz)@WrI2ZsQULJE^0dp`ba4LmBINRvkSgAO8 z{*wX%bAbeb%db@wb<|Xu`DB1|X9p)6;7h><<^=!O#oFBx$>jhCn)C8;32=dcOEUKE za2KGFJXnZ}hYJMZ;pgH7{TtB#GgEmm7cUnCxTI}qYw7Or|9aTJHL&rtwEOS;)gk=q zFo+NY>Zxz8=V8w0j4;sp@4UTj+&vtCMf=YZ{+~zx>+I=^u=(#*Ya4F|7%&b#W{@1P z^wtiRcJ7ugKof`mt*#1^2L|KUSJIQ?TUAYnr<{y5pPjX%l8-W69%3s8@wWFu z$U+4?4YgHlm0*5I85?yQ4M!s>LxX?aRP^9FI)d7EP&Fe5LnQ}$XPBRppN@x*pO1m2 zg0r-=ubrK;th}9}wwi*oqoOTXO2$rC)lXeb+Q?TxUPs&20p{tXV5@BB%4ehLDeDN8 zhw$<7%4*wsAyhOGKC--yU`0Nc;byVD8f>wS?Udk|#5nRaI$5mEF2kC?~f=Q`6>G^tE zY1-I8mEC=G-5`cqO0G^aUQhu=Azhd|uZEMJ9~7*sCn&9D;G=4^ zs^%{3;08eox+nmWEj1PFoDEbENux6_sE>eoyU43+L-f1_WnerYBTp@;fFZD5mg*|DU==_cq^;BhJbnFi{VetLEZw}6 zY_y?j`f4z^nx&wuij#(-uZF4{*ija&r6?sOWak5NkaBe4an+Slkn!Z#a@BBg^VP9* z^A^&Q0V{Yp!?oQYN`Ml$D9G5T`nW@Z;p%BRXaY~MclEKCHG-+xBMcN}JcKN*y?OPd z{B(4c_;`I~wcKEO>OfmTeGMD9x3sqs)Y4ta+8Lw-7`D8iA6N(wAf%><0IBk6N;~t| zDnhMzU=Sq8UI}dHBCiP8IoJ>^Xe(e3hPcV9$*TJ*0sZ)G)m)%ph>@D109=UA8{yz% z2Y2#Pv_@(oy&TkeAZ}X9F1m6qQtF0CIX!+Ubybjo0ML!s#?uFQoCB|wm#?8AFUZE# z+SNk_Y$d1;hG^MXD%cD7SUIR6rG&HuZ55SWvQ-V1k+Z4 zS^K)$I@>u|%G+yNd%HOZ`MJR4c@>Og{2Xl^R2@N9z@wmkyfW$@29^+KIRTiP4$Kv> zXlaO{tpSg!m%akrP1Vg8%m?zd@`S4Us(JA%xbpG)=okQgPu@?_!PUS+$jd>`Uh!Wa zCmun48wglX$I-yrUeL=4>gB-W4e>T`60icip1h&H55F47Nz2HaPXpoW^KV3$>!BGiBpFI)yyTze#__=X;amBBHN1>zdJaWBt z{uRM!sq}s3Tw)skG3JjAC2zh4v6+MSm;H*sE;x$#(9OaU5B?5VvvM?e);n z7gv|#L5mikYPq}kYUZ;QxsVl4UI=<#mpJyWT4jXIH;Uv5S@zr~@0ifJ?Da8Zm!E@#eDbi7IMzn>!K$Dhk z!M-k8QIJsvUx5)}!H1G4t1F3fIP{9~buS+-cT%T+m*B#4!OW~`BS#hra@7{C7fdAX z_MMElq0qBb)R4rjTUtsCEX0BHeX+ruwdh8>D#e-fQ=iJJ6CioeMK5by; zwt$f3{*qA@o369BiN^pU|)#GnW>C!R->)10^x?Qqz1X{O&^hU z38E+v$-xW>YZ8-|YU6+JUR#n?FFbA}4^0EN$UZ?kSkJ{NJRB6pi3yi+K|Dc-qxH}D zF)WlM&E)3%Hgt^~5wRSyom)ygV5=XQrV?%pfu4`gW6L-r62ew6BKHLX>x08d$+7hx zGZk$OksNw- zG9r*NzOr*iQgjI7D~=HOyl8FEk*WHbK_P8Av3M$OdK$mtQ|j38dqJVB&g@UY9sXXY zc*k|vn}T0%!W@5v?J?8tghtZiR2I9hC0hB+;z%x$emvmE)kixJMI=61=PM&v7)VWm za3ZHjzK0Y2(eL+m4~>%`JI)Pj6xATFN6Q=6#3(|N^OSxW9-Oa#JO8EP>uUnLRIZGW zl;G#+wzEA;sntv02TR~@zof9T#3UdRgC8he#Hp!d4%TCaJq)R@wn)YDPZ zI+uk}Bzc^6wtdDUDH!#zC=OP^x!oyjA*+T$T7`YC(XL)7%cNGOFXpJxtv~AtP`;`; zE=oOSm}Etd3588v6f8)m@DyMk3g*$DBpZq}`!H4ZgD}yd-?3kn9xyV03?oj2l|@o8 z1RoJRw+@qD!p!`H9tX{Qth1O_I`xQn!NTf^j@(>9Nf|bT9=Ck=5m+_Ev{3K$>$YpN zC2F>%YSTM@A|n>+#`xpnOp}-PJ1vC@j1nXBhKKA=EniHbOwQ%Ep_>a&mo zdfkqAgg)+Y1K*>;5{yeF%EGfmgaWQ=eeP)UxQgW1i>qw1^TljvbI8GvF{U$Xgak{% z;o5d-risw&9pOUxPF#wC;7|`tGFG+l)}>d=$uWsBt6gahlHSz^PTozw149{F-pP*9 zQgW{ntcnpYktrFMO|?_SDX$+@-Tc-B6lxez(GMrlxEyWi%Ilo&3Nm?O(%$~@9b%)~ zi!`xTJ7o-vt?zZ;YG}*#+ag%_!Jl7jqJt8nq($&n$uInes&*4n5- zJ~iMP*L_kk*sUnLA{^(%%0io><0BLJRRvm;+jFFCwrxgySm@T}#1hq}ZBn*O%jG&K zBlA<@nM!uR{E(N^HCf;!cKn7mQea7AVSp-*lvBfl1kv4v`9l8;JMgS43A9$RdFC+P z06~1YI!C(-!>;5Ft%UIWcnl@0$;Tvh-pYsqAU3SfOLB)xMo03L$`K740x&=t-iCsFT(@%F&Gb!`xFc$dvLcX4ts*Sq=c{KcZWk%DZxE@YwKH|i@fc9F-maAJl3s+u zq+gV4Sxt_6jc;(P3i^bY=LAkzY_PYxrAz@k^^=f{PH1>j%N|g2c8eNSeQUfbo8){8yk!D$o`CMUPj9E;bwvLK( z!~#b3TyANfE~lLZR?&;4vONkWqw8a6k0;0ttGhdN<89PDn;7b`a&>vb!hm72krVso z8MgV>eNJ-FsbopLKwOn~S7c7hOYxPb?}AFAdr1&)DPCV-64M!~wlDA6v)Jj{a+d2K z=L}#6n|8yP5$)mAE>EW2*LcY0&7_fNNQ$17->mPR-ZQQhS~syyy08Lt9%%0g<0*dcNZ}qG%HmWJ@gCT5*i`p5=F^aSg%bOXHhcNpV?Ci# zBJxMrDn8Yiu5poXnh}b;Zx}w*kOmO2SffQ_WjOTaw=4`WUlV}i@0*WH!z6I8K0t}6 zBz|Qen2s$F--bj(rZc(7r6uY3|43=QA3*ntlnL>6U(3ngs}n-^ulO~}8Ui*lDaIu2 zr277g2hTtopG6c?Le}`M29Zj5zaA2ux7>JG<%a3LYQ-*}I03D6JM<#NU?ef>Me`6t zgrLbPzkc%8*vN75YhD;GbI5v`CMVQKy)(2|m{Uu~P3>$Hu;Vi-3-mjh%3qindIU$G zC7p-ff5t~DH9ft`MSm=T+nQTo&9x*Oyoi!5Q(sn}1T3{cZy{j@TIp~q*%-xyPN?TL z=DV68L{t&!7~DBBzDSgfLHBZr_fQ3{3;hQK?5E+pgiG9(Yq4} zGVI%lr%e(CD)`SoVt_E0&^gXJPdHyj&BPLNWU{rp7l;j~A$!K)h zKMbD5hQi()mn0~68!{i;YBLXqG-7dFbcPjPd|7nBBntc2JdlzUEy=9+v3ZFMQKwth zs?7Aq?pHY)(9deCzYw2@F_v*mKdqJ;m?$VpD(6;y6;{I+H6|#_{pm|9G6m&|v)KF6 z4MeEQgpectT~2c>*i5e^F!cfSpSmypuKEP9{g7OjReHv<#9}FldkaxEZBPU+4AkT7 z`;GSj+koF`kE^YMt|aQkONkEk!KK7a^yzfqIjIvGk!7TaxLKVi70ORFt+)3$ST}kW zD0bCBLWxZ!eno1}w06mS{6&7y2ZvvXvhD{4#*26!YXl1>?-CsXv~y!I?d`Q5R(&mGK9i!ptEFf-68?{u=dzA=Dj3Q%o7bZOUw6h!B(mX zX{@@(i+k+!S9YLQ$N0_vEz8~WaYp1k8}Zq!A{9*t?e=qv%!D7nBRVe`VciKcCVfd%OtyvjNQ6qw?x~6KQ;4mpkk>;>33v*GzmTZXD zut=3vd6^HN6u5vUxwMw*<0*{wRT zs@Eud#wXc5M&BfV?gsKyc?)KCCg*8-OhnjgPEaXKmWB8GMjASWn-}WSF^?Dc`7ug1 zTA%&wcbEju_tz12Be@^E-EA!gf4K`|beT_wdUvWCIM{f3tmdy`G-ne2hz^Cm{`~HY z3{67CX9xp~$#4~pzEE;(NGcd%hCYR-R56v%dBds0?@7-YVvgu7Y~X10$?;J6PICD9 zw}pXNhGVFg*D(qkoye?$QJ20abUORUBDniZemjfSr?_luqlmsqZu*YqwvucvI*8DM zhUjBcAdV2WQ7& zUDzeYE|*GIU61#o*X~!30;YGnh8G>b^su1&Me60+Zi1R}517z*!tQZ&!mqUAv@!)> zR`)IxSe8%I3K>hbPv>3~&wWMb;);;?9Ik(`KHQxczA+rKHx+`psz#h)^H_J9+|X@x zF!XxU^9xSfJUx-<4x0YB!!oA@#<4DGd3x5PB0QN$b<7wxB5R_{lvOcdvWuaI_AzBS zmz6@v*%t<1&ZsdCjC(vtjiNh|eb3HYl?}D=xA2Mu_tt30`uV6TxzWyZZKHBCGg$4v zHhdiIRJP3sV`b)acUE8=Jl zsS{zl8=bOhV&ZRTQRG;Qk?pbL$R7>cT@{7&J?5$(ObUVL-WQ8EBSOQ*$~tMTJE+u0 z%f*v{h-^29c@0^Uvq%hciXjnvja^xdMP#2*488YVCB^zJM5%#|{nz3kv`dEaRWP{n zMZ}*Ww1;!duS&0%jBavX)aAY}lYC64TCrQ$17$WN5-mNp@gVG-upVLW?a!|yjATq)-Nwe_U-z!YH zz14F?T&*jpudWw8Bo-1vaWgknHSDQ%Z`kao>Elj1Z=|-0D&9fU#Hvvz8k#$1BF)8}#Uc(w=%bwj4Kf!Lta*@tqh)W7{KYOk0^?~E1S1~Op?%qs zU;*7e5Ke==GBO9$MNp^?NAOkJ{lpZKgZ8P(h8Mmm*1(CnamPLmT{TU0pKv`WIES=)tA`86YQ!-%msxFe*XK0IW7EW0ud%)VM8H0YJ_t0c#{ z*oQF@qFqCGDxw2h#vig_HuhF3zP&VVvCac%3byA^tYy+jid?GQ~Pu^Kq zMWX?vjTs-wvK*F*o*{v!gC(YGL|#Hp&}+O{JTtne+4*M^86odjsOX!@Sr2~C%L9x_ zZbkn!Av#+>Wr$Fae^|%@gAQ1FHgXmF{HS@nic-(PAU0f56go_;0`pMHYv94PM*6^L zOm1K9N9ka&nBZmX;ohRYM0>kO6!u?tLoh;8K`sbSO~#!~aTOl%^=Zu9V{^Lz>*0zo zpyL<0^o6@>s8(#e;fnbBeEl+)oR5;^F`yUI(1ImRr11!XOhKesaen$dFG|vtam(<= zT1d!~*6#={BV@^%jeAaoIi%IKg_0JHg##@@@ipmqUP)vWL1p1iul7Vh)Y{X6??OUt zI32w0TK=p#nHkfR0`cN(S0^)_C5lO$3xSLi{oSjn)l;&&%yq?(96p~1Tn$_kq#yd1 zNl#uLAPf8u{5L+323)NiI05LJunc*!`tm_ZgiBYX#bQM1V-!tsoj{>tJIpdIEkCX0 zQDr6JBXx{yA9RXtFQ7l$qh@XdiE}7nCFk2xjQ4wCi+ed|QghB0^?)!wh+c}CjUJS6 z$!78QGc%$yjhdHOZ#0~%r$qF&UK(9*bj#viDmI}oU1Rl7E!lHSAMro+)QGUWv5Yi6 zj09V8>rN_+vo{yQii`!HhU9>ZvCA(d$25=-9)(V;0D~{7j~XpbO`4P+H?U=2cJ^|< zj-qd&0-*o9yWT#}Zhz84He6W_KIaMOX!VqVISfkrkP~Ja*Q=R#Y0LXXFkt-2l)K*K!Z+v`_b;KuGk8~W z-PRlX83-{RmF@&mO7b>i0VQb#nlrYD3i7v%YdQlV+*km+``fO&I}D>U7wU8sRcTOO_HPaTN)KeBa*vew2uK!DaA>RHf`A{?JQ;^2j^} zA;}0_mC&3dCpQ3CI=SJIDg-|+YreRdG?ETC>otTyg)qv%UIuI5NU!FojT|_ip|efL z5w=YCp@5)LT%R)5zZD!fNvyhBDUrIv$`)tVxs7M*x)xc|kH=7>ykV=Hvg*;^Pk%=BCYA^xW&5N+IdazqNhC2B{Pm zpLy-Z(S9UwCcYPAB@=GNc&Lbe|KxjUrND6uH%WmabHBj>aUg)f%Is>OuH@`(-{9^XPA%0(u;qnlALG#dV#bkis2L%hHwboBeafl0&T zGDv$=E#vH{G$d1gn~rD=V4>EwbQSdnGfm~4~)Y|R%T{ql4qx;I8wB@g3cxsD3~vFK$RD< zCqoS`f8DMSd>gQdCc6WEpzpAXHtNaX5T7=`PF|jYeO)^Zpy)T}0Zd4f6w+J#bGcT_p z^3V2Wx$D)l#b5W>yJ|kJ^*eXet1!|%IQMgMbaZSuSg7A~TiyBb<40BcBFZ($0uvZ) z*7Hy(jaS&Qn{R^Hj*(?!5z=|(Idgk}pW|YK%iU;svm69PC*oaQVP$;IO=j$59dOss za^2Ej5L=YeE~dW-_ds*aZ=XH@7ecv2M2suu{x~@~eGu9F4Et7X*`wuuu}=jXv{-KO z;xKD#x~?4)|1>Z#fX~j(j<2kuvJ&&R6IU>Jb9Nz&n2PsW)jvVZ{CeFDm^*o1VSCi?JZQW<2e)ziNob$9Yqe`RY{J_|MT!um3Rrs*ZX?i~tP zeS3G&Plq%~+`&b)hT~K41>N69m6n#)iy#j(gq&u|XM5lkPKWqtBUSZVy*b(n*`J&n z)$11z+FQI27R&0#1LG`f5Sz1*#67XJNc0T8#_Do zntA7^qhgm!>uWuc3OGTxrQ8P9!Lw%m6*F6>%lbw})PpYM5@@Qz37fO|-@;ihR-fb* z;XHq3ZSAQder55yZ}IM=pJBe~q}7OZBS2+qY~q6E73#e+YQ)=pmmg|mv@qH3e>vK; z`WF9(i03xRt9?#=m%W*awyR&!i!3dAj*J3qRA z$co)>YZi})5VnQ9v9)?1vk zZPnfFZk^I=^I5TgU6-0JuP+Wn_M_M5T$l6=pP zvb8LAaHkLG>pdzcDk9!nZf)GyM~iq`*PrHxOq1K(oSi+%1r^5#PX7e|NaT)$Q)S%Vb++)?V3h(nqrh9{DyryZa@QzSN-A`{3z&1f|@m zQ^#Mb_uqCuDlOIt>NEJR$K%3K<^kury>Ww?>My%3pShs08zSU0GlU}+O0FD7#uyVn0XM+cVvukI|SmbSL-+0lx`e;U=Bdr{hK3??m>mO1^W zkqatz@dz!`gy*0;!PvDK3uOKjXO`!pvZ#_`Dd+*%l`nd2Je7az1yQ@Xjp|d}JF`^` zwjHm=b#8QL@d2M2uC@?qbkg0_uyVYA84TqDgOMAMt&GXB7>f3EM@s)jZHhi19sz~&HKkkul&Z8BE>?8_Hp-zc&PMDbQBY{F z@ZJh4=+JA{G{u!M(?~Y4pWfejaWIKJy5bqI;5U0ZUtgfu$ZaBtcA5jOX|s+mTK&$! ziw&;Jdd>v~DGPx;jVl4d5VDE|+gi?+-*F-dBP7v$Gg<1rEp1y_{yG|4c@-bqB~M$o zVrr;SZ0W((EZl>`v>%-#3=(S^%*pn?w!M+7D6bM*z?;Vh9IY+t#?yVbqqn3eOL*z~ zJf@WkrT=BsSBkdNQ4zKmQ9Imej)6fU{^j4QT8L)>LtK*|eMJJl(VfQ+yjqLI(rc0^ zIso(C+{J#<#yuDI+C3Z2ME*?W>5uIZXlgy`Xtf$j=1fV2oRn@NADoRt%#)7!1t5Q^ zB743ZAh}E1&ERrgE(bQY(6-V8u=_tTG=RN)jb~CI;@9e)L}Y4|@0ENj1A_FQKe@TN zQZ@AKdUsBNmmkW>uEefmqEoz%eg@>klVsLf@D&TZ19 zVM7kyXoQTwf7gB+zAc5j0imFWSa>_9k4$ZxTH@ihwnfWg5bm)MNRwR$NLl_DSa05~ zudjc7`}VC~EHY_M;LqtPa(ZUQHYqtd(1rXZKWYtRBOi#JDY*OYWy0WNLi+rJ#wVpI zoTd$BaV_OG@UiI&nhwLyg_}Pw7RLwFASd6Ro+`vrCpl8P7%7f7du-|b17KLTE#RWV zY5TyQbROGvPXF72(=8|-T)yrNrweM$qq_0# z(Aynabh@~|zaIlQt%w!lw=*V{{Rj~3|BTFg|IIg%u%1TdB+zT;V6qg>jMWI#U83UX`;S=E~28M z)@Ci-gZKIU%9o-HiEmj8j#9~hqJULPhuVG~oKLq#aQj{sjP*GGu$l&RmncBy zh7i)cs4l#dkJ_skJnqs_`9&;odt3>Cwq=^Yvg8to?J(sCJZwDu{fZP5SJT}p7wV?< zXDLCpb-m5s--X$SMuHD=z)X8=)6xhJW{=k0Jv=DPz2|LYZY~>52qr(re|6tb&hp(% zWd`iM$4XE!QF!K7o+v-{iq5-FF53TcSFA3;Kfr_0q>!DB&7jtMB%${-YxaVM%l=jA z?0Deazb$S(1VTtHi&jU|=AVK;BWj;{zI4>p@s~6Q^tQBbg5ASLz8!tkR+uHE6&lY{ z{QK*Lo#z-i+se|PjiEh>pnm{m^Q}YY1>u3d3E+7YkUKv%TmsNU-3zCBd(_eeGcUID zK5#{~dA%{R4@{4%rMjZy|8<*ZT+k3`Z2ss&hMgDh8-1^7hoMbWK44{61S>DBQPMWa zbbQu|6J+`(6EnwJG^(BRwRT5Wx2I$MR1lA*&BmQieFV3q!z;ipvh4!Fs9^=C#H`kK zByJ?VF$UzmiEu$ZNm$77`ySXk0$ve4jRB1|{Y}({I{3kH7yc|GS8M=SM=W~Wi2;kn z3jyQ}e-2rBUS!G-%3FL6-iyJAKP~5b>U?{6gDoC(H7uFPs~@+50W+UN%*h7J?8dTsyngn7_to>ooUJgn+a@tz2xQt4 zm;$Eu0t~*~L){&~Baufx+jrUp@-{9G7XMzN9!8?b*te(4C@9R|@ZJbB7rt7bRF?oo z|E8xi1cQp#3RBGQ?D)g&w7z-C{-a`@OO{MeQZ`~U(U`!V2tv_6-rkU2Nar!K zt!$-BR}}0t8IL~bULf$+cL-jJJou5i$9)IF_HrJJI12iOItu8Hs+!r?gm=cbH?Kv- ztFm5Wr#CpqHj5`ECA~R6*I(htx@>D$xyzx8-8u!0i2@NJu+<;)JOj0z8*McT2R3A` z1_u7XAkm>Mn$i8!NQt!>@aA`MdudLUQdqMlwYG#GB<_U-(_KF7&YB|Wc4eoSQ%rs# z9l6_%-{K|i2OY=KLJ}ly*N6lAi|8W;pO2;qux~P!x&bh)!De$fH4`Ro@D!Uz1m&o* zv*1*<@fM$}a}9C)?WuZB!%S7nZt1HRq=6vNvB>@zKKSxTB1c=r`Vob+hS15Yg~Ec9 zuG|fwYkGOVR>7FNo4cvmn~O)+UH-0e(+~DFDLsCkc?yl8&F!o4%cC3u%;C98ke4jX zyv^Hgg7?*|Z*tQNv~38?o85bpd;g(0Bm<@V@4p#YS);yNnf=9@dEVTjI*(4CLWRD( zFArbgutxogpmCiO@YJj%4@3XHb?SC8Z};0%_zr$XEOtgRCU%*DI+`xi_c)){ zKi#h#ONftuBK|_HxFoIwnjutky%xzJ=&WHym@$~KO-_yHMu9;qf zTOx?pS!D&^-o*YgT5_QMNI9)`IiL?!A>G|OjA;NJVMVxbs+`t+XQpVm>j z)ZM|&)Re6ni?tuiXPzQgjD<_?nW($#qrh*jjVH%C6bzhZquSi}fYZH=)gNz`5vE&c zj86ZAZ^l%7f13$e2!x%z(QD-@$_^AhYC9!534DS-U-f--v@%;fpvC1VNt>U6@d))G z4g@u9c>6gykb?dKpnb>ke?+Q=hgBco zd$Uzre}=d(CAOz#3jIhf4wn}n;8m!5VUXSES=FC}1S@|mdC20TYwHM*+cxI5QaXc?Yhb4I(M9ynf;h7& zF?ET%Fu#s6yH=yM-j=X^ebeV3fkc3p4nA9H8VEo1Zq;nwt!(?F{E$`7r&@1bb{`hC zkSw~D;c;vH@rEs$f_oECu+8GUxZIo^3nW>=ID1u_w3{0*tUu1?YllI;<00^5~qHWN9)-Sl{O{gOC~HI zA`7|On}ZvFK5!(TT@!!hH17~yLk;x)`CAShVb74U^5}oF^9$rDgd1dVgp9dDMpAEJjp7Cz}IQ6*+ zmAj`S_NrbV!NUJ*b>6bvc315Xy#k3)rl|KyeQ0cQ(chg+LZxYxXN@^N2U&JwJ`AdH zTGPKdmI}Q8Yxe*WnX$Rt8T9#?r?oPa_}j5*Iv+Y(68fsNxK(_&HS^E)Abz&@g2VlT zUXVUxsL>-Uf?9b!AIVvI?kXvGOk|_4C=`SRwUY`NmOx>CuePpHzw4oD z=Wv-rQ*uu8-twTk*4~Va4wgn1rWzmdGVK$`%Y*jIpQpryF0v_E_g;`jh?^co=9Z`J51N9G);ZOJF2T|A|>D5@&~>2FWH`VZdyjJ(!& z*cC}PjEeIirPrPSgpVn+wa4R0P*mC79=>A{Zm{%`Jt|A4vZ8SetQe_^GoAMg^x7XR zgxN(Tx5D&yU(c!)4H0PW3ILRrzTI9p*0 zKp@jRQ@wI$F7?aC;Tv~lgG`99``~aQ#OTz?CD6C5?)Zw7 zt@O33(}-l8DL1XP2AGxlJEM<(A};ob?LfA}pZ^Q~KTr#>O~1YFlP&m9!UJTc|A>>l zpPvaxdZ7ZS_)v#ef!MK4s~4fIw}(f|{+<9pe>F`r124XJn^Z?~ra$C1tnl!HwCf`- zXx!ap{_g~3IA2j^P}uj+`T-dkS&!XF+PSaF;FHlgfODR>jlj?e6}J6_{KpB`6g3!q z+(6jh5PsU?t3vt*oL8>>m(>0IIkm({=4P4h^VE)KV0va|=0vTXI%wy|i?5Z&jd4H% z8cY9SYn=Q^4sNJpU=Q|4%q@MuwcHAQ;I}7KCj?Fu;<Sax2E;=^@c#A zXHFwm*tZU#gqd5x$6CiLBj1m5CgthkWwqvWx16D!I1Z~W=j43K{G&6)Zv13 z7Z(FBzvtykOJ@2XHuva`f=w@CgYj_g0KCZgnvG3(-kH+jzbSVqiTS@g`8U;dxtnaP zt_&9TPaq_|l{?Lvyl~=h%zZJ^v5zGF$IS-VZjEMUrwh4R@m$L3^wsOI?7m&w97*Tq z;^e#&$ZS^PpSl%R9?Q-mY8&Q`j2=zcnY5Jy z{3FIcYIlZ}8r1xc-Q`o^Fy}vl0I)%(c|-VT_T7EoKr9t#GXt6_EatJ5iQH}MYQJg$ z2t@V<=Xv*efDJW+L)g;6UjaVk$}nU_swhsJ<^O{b8Ty$DjMVqT6PbqpO2t~aSP?m& z(vBo%5MTN@SQn;&PtLIlkS)9$jZN(*S}U9UDF0k5=D_L3=1S96B7FxnPh7?vKB8N| z5f|Q)7acU77`|b=SuXlVhtS*C&XX^@_0tX1KsA3hU2&SrY-F|o1d?Nb_{s#Z=R?M3 zxtmHkf0v(1l%0rNt;b`8i}`BWZU(O>CMIsGk>l6tQFRUt=}!$_FXW|kTz+jZ$44w^ zg!k*{f7(q~s$H!Ns`Q-KHk>uvefkfAdm5to-}YKSJAc z`(xH((byU;995*?|%+AT!s`@)(8OL50@Bjuq+* zMt~&dGfO|tfn>WY-n%kASH;;Kcf{>=^;r>P!DKkQn$EVf)&kX{`O+k3_{+jvj z`kaZAlk>-p<@Gg?h?N`F_o%o>Xoh7>FSxJ+lNOuwoO-geR#21&jhmm$_IH*LLmkInD~kx{c9 z+rgi@fLV9=iD6nEoRLqFfd?xw9@a z8q7mSdhFjl-8qFgvru??9uZRVtbSJEYX0_>=5|c{#(4us!Nox^=fO=VqgN30bkXEl z?4sFn_TG5M9j}=G`PcHhO5@fBcPF`CJKGqw`6a*(kSjRy>+8~SmGG60p7N$#{Ep`w zP+4q>H2Ogee$+_75@ln%yXVf%&cB=5>`efm;{@&l-1dVYMa?IC7gKLQsjlj0%@%*+ zMQTnd4EHkxwfzG4N$n?Uq_)wpph;;LQ^&yAz2Y?X)P)vRr)57M00t}=ukl0)TYvRC z-;Ek9Eya3YCs%R+7FDrN_@r%G)`JI>-C8$G%PN(@r|0NiYO4TvUIL0F7eOz(pMN?o z4BrHsWDFar)5$9}u$WF3sb_nF%(*pYF@ZusdqqRXA;Wvb_m@k53$w>iFQ%-^(_MzYcn%C)^6kzY_Cw%m$5ioxe@lfO}$xpJ8!N03u1{ z0I;p$_y1^AAVaLY2e<=kclSTt!1VrwZZJ$7@s{voW@g3TdY&yA`55uZEbFlpabFW) zSaL<9nOci=4o^K|k%vnI$s@+4VE9jta5awdsxt1YH}^mat0^z8xerurD?@5U(3{!- zFg{zDpPw&68Fp4A9RwYrp@z^#4)c?W%q>*CC_qcXKuc&?Tg?{@%M;i*a-!b9yxu z*C|mI0$)1hJmC}qljrb+Ts7FL_gr};P$IajlZut_pWbw z;6Ux$IS_y9NUSWBl?OWk8ef%l;|ow(&IE)l099XW)&e!0wc2^-dL<<#kpu4=^5@?g z0Z_FeARsVQkT}E;a5f<~!Dw*#Bmk4>8@#?Zj7s=uXQifL4W+JVNt zb`g8Q<%RYOV!1)6c(yasb1K%PqEp=B`uT6ru(=n&`xZCW%~c(NknYAe^S3E=cYSHa zuP)%>TOQFO&9AJg(hVP>DbW9ZvJ`2Z*7y;ZTtlgB{85NhvuCmT8ihY0T?$#PZYYcs z^SWg_`NdVay!&v9YX_**8+dspDHZc0`;c5^*b7WhJJ$Nk;6dC1tK{eb(pS%$hm61H zhiQqUC3-boVQMIBQTQ#>!gZ2>*Q?k-xQzOR(I{LWgYJEIu>LEcTb>nytnu;oBj@_r zapS7Zav4+*88(d!^|WM!roIR88l~OOROr*l=(3O?#;eH2U?;VrU~(T6&c0bMmJ`lr ztxemn5yR9t{06<@&-nHop&}5cVTiV4d{RHN)-m@gRaN?sT1goBH5g~VJ_`S?)ojGj zy7H_wQzl_I@P@FG>%D8);f8-+Nl9kL1f>$AFJC`M>q=u*e6Zb>7B|18KzG@nj6zEf z7@5?KxSm^t*@y#+U97L2GAFHXM(nEBs&Q5NVWEyRemge$jbZQ3bcWpK0DYOz4(idF zSc1$gMaUzj%FVh@wcpo4dDMnn@`^s#b30y>Gcm&Ush6@AL1jYiX|;W-1H->{pI<$F zo3i(R)%2BNQN3T+#DK(rbR#+9&?PM}gmfd_B{?+G9nuWlAtjv>BQXp)gouD3Aq|3n zLn$c2GyeYX`}thwoICcu_gd%LYaMg%>ZYWfsr9~-i5(DtBW_pDG42-A>g>w(l2eRA zGdJp{csDDw+{j1G>sTLm6;9QhG3kWsZ-!~!2>m_&YNuq2ey;eZdQ!{J*xPupwbj4vl3K((IshVCTTt=S9KJBuZU2P*#7O%;CfVHm5jYPu=v4Vwy}d z1u_EC*FPZs@~$`IaNCRxd_|M#;NbDU)~!6gDP?OyYpe}b)H!oePv}Y3aeL27uAz>k zV0FF^lnt-LLgw+78Mwg2YNEelr0B+T-1OSzxRAp^^@-P4<8~7 zvAME)dqsshy9aVfX?-J!0ey!1PT5yawV6l~v>!JG(2nP5?O0a2HDQ8+zP&XRcWe%o zJMFYd+ft;RW-d7?JvVe-kKYEu*YLD2f8kg(wY z@A30z^r%W@md{zzDjZ$ZlE4h`o_T^CIsUr+a4cd+MyxrDai<&?D0371*?= zY2fs8J)q~BSn$htrz1dH z7#0@8?LS`tU7WX`F4WBYxmXxA`LK_8x{28rfT6Y}ez)36_Ip!Dhbv413riES77dRs zA-2RI`{y`DPjZ-p`cgM3TZ#s`umHI)=3ui#oPVQe`=I>ojc*96mSNpKAB{z>zi4t} zS|uNM@jh{@Nqz_@j!mEMoBzAvMSvgmZ1np+J#l@pAh)E&Ez3u9WWpXayi733+(a)S znH@~Fhqqg(to5JjKzR?yERyHt*jx04=7k8`qEFB5k|HT~>D5sKTTK~m!A&u1sHT?s zHnRFx8wxf~2N>>yj?tM){)-lEoQZueA<7=p$cAaHt*_kD=e;O%L<}KT+@7Id&oYkR z<>IFxW!U`>%%Wne$fj$c#`mC)j9mdD6r3UJ#f}dD_1XTLK1aGV*;;D(ggJsQq@lQA zHW{h%(vyvg3yUdNe{Smyk#kPw^CD1IU1zqB8NKjDfoqmwZf}9BgT#N~PsnO+^k_+K zsFGVhthY$xr%}*12h#fHT}L*PO}@ENzNyhtRLNkVB{_Po5Yh9* z5||<%L5tB(8|36;LAawe`sZ(d?fYTGRcWCgDA07Wr{GnU&C4OmLsRRSSt;TuHG)OE zE+CF6KWR=YM7=AA_S2ob{$-hxxjD4}dmUI{cb1Q%?a@;vfUu{1RqI)1Y*3QGv5c;4 zYB$oQ;X55}6eI{(AEWr|m;Rwwo5k;*H=$CFV;O-P`rmT?H&uq%pX`meL|Ltlr`ve` zbfeINBgc7@A}qlIZDoIM8Wb*q(n@?OUpopwBk^4<+*qunj3I_~el#I~(LPVd-o@VE z{CwkGmYdq;)p9sM*wS5pZn}CGeXd}XfSb&b-q9b|N3LyY$#qBI(A#5-1P~fbt*?M)&$NEwOe1sBA9)6_5ALR z|JlO}T{ijKfB$SABbra;`4j*TKPH=)J^-`z>Laui&~hx7Xd3j!a_ZX5&-ctV~x z9m-EDn}((l25*nSAq(VQyKO{6dSVhr3dZykyh5BuxQC=gDf0V#ezXD41Bru#KWQ&e z!}gTjQ#vg*7M8S|LAp!!r&01ovT^!8MH!j`Je&dyXb1QN_>}=g>VO{$jAPTQjl`L& z{7={C+qLGTUwzQD^ggb2b{fpWPGn{3E$1Kby}lh46e(*%s#trniH|D`IRC6Gu@IeG z2zyExH7x@ri%tMc^CTm4bKxNgyFUn|hys;fia1m7X+am3l;2a`YLj}p9cvCP?SAkZ z&Sx#hsnkFjX!ZLZd7a7GBrnZOp||jd(&cW`|4#?mw@*j{hkakR%+o!nlaeCUU>4OhSYR{G z`I2@@79(o`!Qnqz$s|H~51Y&lD!6MG=0QeVWx^LdGfws|O|c&)fbyFcI05YzDH+$k zxZ$V%B8L9k>Fyj7pUe@1g$s31%tc{K3pN`jU6Aeb-lDw1g*i_~!tuc1klGtxd5m%3?7*G5d^ zuyGM?){a>49`x~_lC?n6;qk!AmoPC#mKd&&%8rLW?Tg1l8E;#}QeAjzbt!o)+t082 zsWdO(l0ruIN*&f1SCqt>(*_h~U%X*tG7orv!X@D-=BI5Ha&Ll}E2oI^ADOMIvB>4o z9+e|u3Jh=#uFXDS@s{T+R!ee%IIgSf4`_)cs97*Y2b{{~lonbl|7n%C!RKUmTT=Sc zvzpH-q0EiU~mw zTj!o_oWv5YWd*P8gXAh?2gMp{#7mC|JHp5W0EK^|x7V5(w6TL*MU=Sr>7OeocJ8&uA-H5@x_wOCjUiP4cIQ(Amo7orEvE*G>b!O|hZW{P>Hsd-WXG%=!OCpdro#27uC3$*~XfI1$ix3aDeCDvzIE zNNY0Gzm*210N4m#w{Z1mVCIj=bxt6&&^OwYQ|)|zO}T3%+%B6MF@AnoMlML|G)`B& zU2wyxVMNoeiQ4mO(?>cnhPF18{uwMBnth%jZb9EZ=3A4!w&Fa{>m{9{_zV+3SnZG4 zWdKY&e``m-;@{1Fuh9u?O0nkQDN7E^-1QOf$mxU}`~qEfzZ|5Ng%Ud2JrYeMovLYF zyf*u5gsiYvZ9XKEfJ3W()VGc3_b)8FTO3om!=mq-^2gqki-edmo9XLWN1J!gj_9Wa zUOf{`j}9K%+JinHb(`K#t1PkbC0{tHjE{sna35U{ZB(RcJKeOdQ(-L>g@q z1GHyuSkj`Xp31SSUC{WX{MHKVfMtLHE&~i-`F_z|utPt4F!6Zs0=WC3E24O`m7RJG zAIbveJRGsV|7F_U$^OKW5@;~&ndG|1U7>(@d2O;?0zGMUG`-Qh8uAetX#~!IFaXQ$ zg;^Rx@#{ZmsNe%JW%H#Z&uv`&n_#w*y*^$G8ULk1*iAHKi@>DV#lnw*9eOaQF>9Sl zt8ljPs8DHzs%vceu9$!0@2c1Jft&f%5+-lj)j|#T_p8EE3oulU;QnuK@(!DUGw|xK zXf-vnF%7=e8z%lc_VR>#>3SRECzEL$We?Co>I87s9+}QPj4FnsqJyfhkmw?p3SifC zWEYB2n~2D&5P~W=(kTG;AQ{8c{Td8SRd8G+_jTF9Aa_;g7pG3@A8$(dL+>G4U{A2g?`>jMy*Sczt zB2lq0DNu$Hmq^)o+%Ci6PR04*os4~ZlsZnLmu5L_W_mxK-;4X2Pv@QWpl7J({*KR3 z4M{g&f5bxfo`6-)Am1INQ5BT&X)@1=A@Nd10-QyP1-s1rPVDqgJZ?GN=sMoK<{P4o zy&(w3IpS7L>&`H}+qmd8S0z1D3x}W;pGe(DhT{J|zU@5v8p#CDXTd1>r)MjwMZI=? zX)?vhLbv$J%c`?QB8C6qWloN&-G+08Qe$=!Kks&(-#XVWidK z3sqmBqd1Cu5FmXde>;_&v7k0ov-edxyroyu^Yyx+USOI1VYT zC)P{Cp8Dw-0|u@gj*Nm?Nrm&m%!Thp2|WI73>btv@Lc%I_c22*+xv)2`5)z?yW=Hyd0YH&z;r2-+k_rkMQZ+-g&r}Ao{)Ny4!Mf1Ruu~* zK7_Hl7MW=!BrhUt3GVj!oMpJfogV2BGNy+R2W~7Rxowf8W1KnSYo7?=0WCr3Mc?Tp z`Y7TqI2|RnKbzUINpn>P$oCtjLU@2d;ds5|SrcL({z^iBX=-poa)3AH zT#tboqR%^J!U2X)c)>w5LsMUIJ}kA;(P9{oCx_(IPly#${@)Cs>P+uBk06w=<(+3! z#1&j}V6QsS0_S9xeO$BqH7?H3b(-s{TL*fs!#$Xlyq14VL4J>V_~4;Pu6vAq-us(r z6AD+C!sN(dW!Pkfu9ZC0_m6AhPmBi!;3#;ymCS3m;?0XvI|ljc zSKX`L-1=C3N#BZo&F2`2mS0xLH{jzciUBtPlClNY|0uSrJX zJZ8Je>&}=eTuz13C;~7nTWwN^+bnTuW?41CXZNBcqqwn~k3(5|=^-s9Es)CUOUcqW z1^f}OpX3zJ#j{Q|dS-e*;7`MqvIwnE&?UNsB9%$UMkzuXL;|GDP=Svl$iIN7y}X*hkVP9SkE`4Gi}_)P`L)%Ke2sFpe-{HY*thq>g5sA_|oFy7T=k zJQ)#r#ss~%i4pA1k^LZ0mNpsNlpOc(YlSJ&4MS_iJ22|Qf`WbkY#o-Kig(3Wd zI%MxinrbG;<9E}Xy|!m%U?We})cRU4WyR>t&H$9!j;hvzqhxccTqQ0IXGlv6}K;$|F@7Hg(% z2NeRR_P(9J7yc@xManJa*z~+W9=dSR-Y|XSGb4!7T)OXmbvL9}?8tQA7gE(^=#h|O z)@gM6G-vlG^5Ho74I4*^UF|zX2$5?-r*=XvH#4}mKUP0VPTeO&kDFDWMA#mW$@|DA zS-W5bTQNdVjI{Tef_W?7m5M)a)X)ln+E4!gUt4`3TkqQG`su(AU{(QoVD=((aCU#- zeaMeLGMz;E6={^&Pq4MM<~UzzYz>qiXFt=N3@aPV=4aUZLX7VhLrR#wr^(RY`lDM5 z0LRh{|D?BDo$YyHw$76AU>4 zc^vs`D%!_6`uqWLJ#dJo=#Au5k0VT!l6~^DyL-)F_0c=wwT8g6^r+Y&W6*Jcj4Ojd z-&@JU(&WydiAl8SeFj*^)=56mBEN%C!TqL-<_#u(z3j!H3)kT$@BA2q@h#w@*4LweAjzd)<+Z zv-T{hzt2H0qF+Y^X6g`ag`>kv5M7O^^+&z>P8uVTr4Aib3$bm>n5j}&8cv>kqnlV- z3{ge4CDF~ls7{$XlU#6J->gMg5=aGm?E5)fC%pRl)g#C*Q-|dRV~R}m>fuLz5MyH7 zwCnZ@ost|x59T=;%@%2;QK8_sMgW7TPDg#Y;D$+L9FS9=H#gJr)J@$aLnLTTAeRDc zq=s^sa;UJD%_&0*l`HXwN~QweC}ra5qj}25KVKtZk6#^BGbr1Y2jlH@5_KObTc{Rx z&}S+OKXVs>4~4r2n9dyKT(+t)4vnBlLRpVJA=wkDo6))kQ|VC`djp$!xu`ei$$~xP zhxi+~kI2x+(w5Vb1Bo5smEMyY6=#+wOd2cVXz{_n316&{n`VR_T@H6J;A}i~14VDE zlNfu?c@eu=)R}z0V`dtGpN{R=_g=kanK-3MI_t@&Z5sv&lRZCxoItbc@^$t1$}|en z*%NVM@8+;+blNQl#Ne-!4XNd%j!DuCb-b}qn4%$=2et~EoI^|5k{DA}(p@^+5loq_ zZ!#j7b5UBhLw-U`>buC&t=F5$TwmA$Q%pVj_TkNHPtSNoui!)4n{GIz(}khcy3(T- zwM!R-!Cxv}7T(of#KDsfS);q^^(~fQ`>Cyp#_Y5|fhq)LP&B8P2-@?du@(Pt83$nS ziS8BEDr_xrlbU3|yH--4YFJq?)4AN}r_bQ}%o`3y2`fMrOhhm14ZPUrNChW*;%2D* ze2Wnm4#C*z6i*atqG!Abm=3{R?7a_$iWGLb+&K~^NsQKPsx$g*9e=wz*9(Q>*JoOY zJV#4|o!dTzExNrC%$`tYn<7V=bVR2wQ=iU{zn~yJ(0r^y5I%$%4I_+K1>Kb~Jym!+ zU}NuUdn43}uXW$@9;RyYSUlT0@~d){(g$g1&K`x(W~XZ+EP@DjyD}*_s8j(!dLYY9 zk?P_mLoz%kpYi$rz*O5nMf(Ho4?8t+7D&WQA5TqjGHoDr*$)^OCt+p4k?%cP&S8s^ zoQ+cjjaTy#x}hN8XAS3x@$bi7tK`|s7mN-0G#%|%>n%sxbNd1DVqe^h zcHw--GEM5uR?O?ZsRpP^WG7>sK|wyl1MUkW<-)p_v$di{0XB_)7T9}0c}||XXDN?Q zVr=&SdDS!6<3sc*;AC|xo172F~AB_WTmB#u9FC#kjtA@jnPoogD?O&{L|C0hokYmfU&PJ;?#KYJX~|SijxD>;<7T_txvM^Ur`9Zk1wRcj zx6Cg0_#d(KZfwih6V%8;Gs-~x^No2)9lyVLu6%@ig)?35BKL&C^0n>Cn)Za5Ao+JO z%81XtLPmoN6DU@DMwes4;lCg6g`-Qph{`b@{w|*`>q)wvyV{MWXesmNhFxTN>@^(I zeazy;%iWhwkv{9^wJBO~ZYz{{5rF-WT-P7>h1tV*1jcK{sh~7Cv(I29(b zB`_%pLPw``{M$+G?)4bNy|kHWYT)npZ)Y)xaJ_k(!dQC5747vpo6URfMwwf8P9QW# z*QLPymCg-Y{1QjS4hyGQO=y_L z&s3|k*UrA@J^XuH!=-z`}zG$t~_2n4APA`c=LItSNFWnBu~IElq=J~D8B5nr1sV5ESUP>7*P^C zmK}aJPtxsBoVLl$=JHLTY0odT70*6yhLCXlji1$AavyMM{~jwKGiI_v6gL9%g9I!! MWo@Nq1)J#q1BBo=VE_OC literal 27391 zcmZ_02|SeT_dh};EuScMhi@PIASn}+j--IQ^o$&-;VM%pi2?+-u z`v4cbC>HPIC2H?sAB6XW-#y?DaFqw%!NtkN(c#aP65v;M)y2DTS z7+wvH;l%>}qs2whQh%=S@rU7MdR=x6z~j|DFKhj|#^2G0;DY!3a~WDfR7zCt&mVk=UXK6B z#L?dc>xg_RsR2@RaIwev*n>>OUH%b;o}IWM9;YE=WTfkGIzUYqhsMJ)5RrM>d;BxV zd633wm?$!cteLxmn>0Q^^Sn6vyq}ceIW-e;f}^jWgp`vnT1VACAch;3X zYw2vKy zo)qzdxx1v1rH{6(JZu+E##I`Ac$|k7Xqg6@6J?#{B&~>E;+L(Y%w2r6JzYHU8W*r< zak6fP=g#?QnFJ9o5PfuPtJGa}RXEn>lOi`{=o!GnJBXyddppui@k7t0wR5 zps%B8Y%1p`L6kk`c2>vbvWB?Ec{!Yik)`9=^F&jemW-E_g}9$Kj&RQY91gFq8=yrX zdTIE$`Wb8D{oD-P)wE3XUDXIgyjqaHzBkU;$PKIMVR1p*$l2Z1&BIEc7^LfYURzC9 zL*K*7Nh?72{CNV-Oxr`t%Gke@fsS^E(8}>9W6r_V}cX>1!U@l)%3;7xQioJ zL6mfGm+|nnk~cQQo-?>0DTj8GaB`K9*2i1O$!Y4JMZ1c72Y8s8YZ5eFCGa>yBP%0g zQ%8aaUc$oH#Mjj8tP~jB8TSCMbLail0xjWp^kv6$G8cSZ9ds?^bTs_DWC^-b7GCxj z45h)6&$t`gQ z%R9(h$xB1nctEGj< zy1Ph8xR}XYaL_Z+);e#fZysc6C40fp(#+i%Eqz8WNdLT~(>X`^AY*WKXUsfw9poj6 z@N`XMALJSCx_&-p$Y$WpTx5;Fpe0P*i2jo5Mivf^7mR%a94*Xryd*D}oez=^wDNXy z5)btAks;Xono7xNTDY3Knx64-6%UY+G_$Y@^s&&y1zsk)SjfA;qX@DxL{lp>fAC#r zq_Gm#Dhtmg?In)TI((h-v8tKB{k;l<>pW*L6nZVnn_^lL{D(4wO<;2 z4S(k8?%Sq!68aM?iXK~U)O%1i(8w8oVpBvL`eM`Av15AD8=JSDlTPBU-SASQn>)X4 z{3i!RDYr!K=cS^NZDen`(`C$uYO{|GLIPE+%}YrUcdUhoYpc#w*|y?mVp(I|w$!=| zfy5`RD>^e%_%!Nb7?tI=357b<$C8$pKd$3rW1f+bQE00elMz(mX_xfbQOAhflZZZw z&c}V1my(hir~V|8R+vkbVpQhgg^O_O1xfD%D3q#76D^7*^G?_0u3OQg&>nZ0NgBSm zi%{urL;Jqxq_Kd#MDAb79qX3ku>tbd8ojZT&80!yN zHw9+3#-lYW897~(pSXmm%x{za9@ws~>3NxYqBwI{wYckn)SCX4OsQoo7CS%At|oq4 zWj_j)e3rQrrTxn5uj28tgWTNlQf*uZv3oatcuT>gTJB5=EAH~>I#rgZ+uF`9X}-}{ zfltRcI;7;_lNRT8?lsMfCq~&vbX<$9T%``qv*w$JnfT49_wG^*)D?@(YL6QlbT+FA zQkf?Cv>_tNn)U!q5tZk}HHP-3w>=c%tKP@?)#Q4gYta#%BC95ug(fHCn`hL%PH_MB zdaxOVBC5mGxNZw>l|rNKag_17rMHxFSRE77<|^IwEP!kSCQD&rUEpwy0Q_ z+2yo7KRv*yqSDBLLRlgM+%}QB$1gM5=IUp4Z>lWMHN7D3nPt}^ z*OVo*97o)IP2&bsoD(v8;gr(u(8^@r#1ibTpp;pwTqmC2mkXSE7U5#qpFBJ~8QJWp zXIq*SQJ3cQ6i@7FvzSR01&xu<+=#8i^hOmG*A^Yns z!FuMIMg=J$ehPewpSNBr^I~l2W3@R;4P(BFW5&e0w{JvUdGV#3P4yO$lvPlG)-nFv zlhgL{*=Um@$ClTaewV-U#ydF<^3{Q2k?0LfOmtL~xFRRYVm~sa+r8MgFWX+ivk_G{ zK3g$f=6_ZmwKB9^xk4oHY9U+sZEQa_DlJ(+K!E;G9rgYi^00HDwZDFO>yUY_O}o!& zX4oe!Y}DRKWlr3Rt&pX`YR2Zuo{dtw*-*Ql64|fp{e>~Csj2Zd6AStNofS%s-+yIx ztH8xkf3iR0PVe)|u1Qy4U#Ywq6zZ=tAltw8ZMte>bx|~Fs4NddHO@A9Lv5E(WOs@EU;*LUmFbRacQ2UAs~Dt_hR)5_Ec?8#Q=uKn^YifkywXuiu)b zj#Ax;T-C^1cL8kd$b)mj!SBij$N=#V@Tm5H}K-}*tAoolLPC#;qgcGbNNpseNOO?4M1H9g!R8ae3Ke?&gjYc z*Sh_RX*(rNQ4*P5lbub{DNWDMWhSin#pG|g@&Vbg&0BG_%xr8jD94cwoW=&haSXMv z?#moWQU!x)lxL#EQ82&P?DjRbIc*bPTfEqDsQN-L-d83k=2I7aO5J0#u&JA_IJdw< zci1CFYI-YvKJiV||y$w~jNsl5!jou7Yh5ZqAKljmCW=vlV+qe6JH3`j8f zfN0bG=LhG#-tnR0Fg>2MDce58rcYD19Y+yig9Yzh6a2EUP+}8fdE!CI2-}tWzcAFi zI1ujq@OXPWm&g@VJs&dA-%|$}L{iU7`kY zyFa_2F$n5e0%Q3gKfl?oO|3Gr8&QwDCPD4B6RdK-zwH^{Aef6>a+k3@FJyh6#p z=KUzagCLB&-V^&H=PA>*wVN1{sK4(F^Qm5rj*ixO)EO7k;g3?S3M7e8nS;xfQD}%F z3D0{Y0uA_8jX>YDpLrIa#75clZbiKpyrExCCrVK2y?McTfTwaA+)#TL#(Mf~+>cza zE9I{ZsGl&1QcyFOP#tZLUj=8hJoG1FWYagy?rBoIVzEi>aZp8`L6(q?`_@e;?V@v; zmu1U%4j#Nr-H(dTAhL^l^|xT6HLqK4L#aN8i_c{quJhiz>FTXpwPfIMZ(jU?I9;jqyA=7*?1Y;DK!PiMYXUa^&K`e@bGq&Hdy1*W20pkAizg^z`(!A6bbqh_;PwsCE4W&3XaBm=(&w!SRcK z``})os4LwdGXJ_IP<_wmY}}yKmvpM>o!13_bSR=we?K+K zZjYV{ROYWo0#W4tKeA~C?%+ryop|EGIsXiDeEe=wEW1$*$oDD8_r%tXi+6l9Iw)}@iOnEji}H)GNRyIWCkYvFMhmy;j2l8aZcM29q{H9JsV9J zJUp~=xT3?olL|^Z@5i8}F3ts~yqx9|r}?eJPN}8a>Vbrp7NQ^`*e3t;r1vWrg;_%-{E1YA~E}+xtwvW5fNp%8#Fs2Vm-9pTgL=`J>R5` z+hcnt{U@Vu_6~%`@UnIE%WF>61y+w|u}6((6d;5P zyEe@isT-$@R0JQt|Jl*Z85IgR^u3P_t@$}h)d&DF%?|?Fq!S^^=GV4ss<8Xu?UCpW zA%C)Y$etjppz!HAjvK=Nqo(JfM>4E(--K`6g$6B!L<6u;V+ot~Tfwa+qDhl__`;57 zyY=J%Y#IwpO$jtNABfZ-M=mZb$T8Hof0pGzXzp3?4-Ax>O1Pp2@k|i(HQS`bYh#=? zU~XmOF4rQ9m^2@?UB|X=EV4RH_NnH=#*S~txnCFjGB|jIp?{n<>I-H$G5+h)k{si!Yx(jeZpHfGk}qQ4n$}sZ_Cq^f zgNsp(0T{}S4Lr)BdU#gGIn5w~?-fC`yG!-ar?UtqtH=&m&!W5^+!o$yZp($8h?#4o zux$sxJZK{4_F={tT0f7Oe7l&N*kn~?Wt!0vC*|jE zBX>R4Ve7T`t0__``pYjiT{#X*ZUzNRkb-+QIb0k?d*@4|2oT~{qb^(=Uq8@ zc^&0F@BM6+@kaTy(hm09Al$SwPc9{KZd9$haWdE{H!%u5Z)s-sThOH%J6_f^^(y)s zSfKm^;SEU0)+B}R8Qj;T7$?*}EN+cq8_DuwJAs&;^0J$tX?6D&BqB+8Kynn|>qKX# zvO*zsBA50|Pm8bZtl#%2Kv1M}@?Fo(1E{CNL=pgRYg&Z`%~}<8+o=78wo$gpPm18}vgyYT=ni=M4}UNH#@Zouwvr6b#4#1WK&sD0p!L zM~R@;g3ZWK)yF_vxcA%c@H&LLl#Muo(lK9!C=u;)$Ux=gU1Z}@9zJ|cRf~;@>7+rr zH@`R=R8d(+gB&TveEZ_>{i6}`jwo&Jf0ELU8#@ceJSf2_eWj1aJ&P4{TY6i^0eYm2 zTjjp1qxCZ<`dK|)bZSUNTu?QQ5!|;tG2WCSJN}-H^FEyzG{DpM;>nXk0F}s$%C6P4 zOWwS71+rQSX*?y8`~tDG+`7d@lKD4X4Lz-YT7Z*>@w(vSw?2AilI=^ekpAuO;=`FKeL%SQ{r*)qK3&6^ z^mcjStygy70&}t~4@_e6>qiGJn&eZx5e1DOR?=Yc3v1&x)>Z7 z7-%KQFQZYo>v&oiWz8+o(ZCAfkSF@ZBsrf zE+dne^IW++d3Q`?kEg78c-S*G&||gK@tv2H)pj`t2DaNid}np~pvWl;hCb=FPSSJP zVQ>|zK_$O=t`C(>z2)ePLcco>A*G16wW982X)-pkBUV5UbrR8A@XfRZ>#4M+yuCSV( z8;_}EbUg@^Orl0m>!@w1x2lHWse44=s%P1C2dimm1|s(mLy;M1a=KTyd$%Ge-t}r> zU(^?JviXzqm)zf%2jT34Wlz6qa|JFZL9Xo7_G&a_2c0&w8YQu3*=Vefk59f`7ON4c z66w=o739*0^n3FKS;a&FZtnilz}jmd@Hj7d_r$3nHjA=ks^{~SpWH3uO8Z~8IKQwK z)qUhzR&A#>Uz+EZGSNFsZgV{y^uT~YdqmuGEF^6>lo--y4%zz2x z!pzJz)t3fUD0>i2sCFtbN9AH5t$da}e5$+r6sCrEE8z=Rv^xQ^b&hc({}%zgbun59 z*JGScsS^v5E#5Bc>@c%w1@bV59RcB2L&p5c@9Y6jWv8{Bd0Vadg4O=o3oCJuH+T z)?{?!bUS%^*l9uO=j!5CbB@rF@^>QOy2C*vyR zvIuY#2Cz9w&G@_iQ8K-eCDVOjsK>9zh3C`F=yLG&e0>E!@Zf0>yyCEV=(T7FYYq(`kP9>ke`)p$SAB6ZWZO*Y<*}xQ~0$TjsnJdtt#oYPKy5#td1AeMyO{v768%5KDQtldM*G60Z zU;&Wpc~Qt3DFGB!pGX304laDmYFt#qJpD?(7`~Th`A`Z-{66;h={V1Rs@FPEb*GrC zHgDZ{>TI^eS*Y!L(K^~*ryWF129vwvE>QzPcDDng>DsZXx{zwDYK@8zEodu5vE#I5 zcdjeG;P!1FqiYiE`iX7UFW4dv@F%qgzYAe-PhSX+g-{+_kwY5~bsMZ<-2mD;X^yG9 z0pv1N)+4H=x#9TV%Aa|g_ZD?|h}Qd@aNh#vnQ7GjOGNN6Y@%6C1LP1|Z|9T~-|+Uk zaDqONB<)0#Jnjh$J?y=RnB(WAcpxHG?`q5)-Sl`Zq_3D-0C1n=cnc8e*jx~VyJUbI z0AgMdhLxz16GWg9CN%^46Ij#t<6h*oR@ZH=PT4i&lU#AZ2Mk2Jz^maj2PQ5AfD{QE z^?Td3_x!s1L>ZgKxzu*GN2_&BzFT*8falW zSOWhiNe7%KGXuqe&#N6A_$?3%9{?PDzc7@E@lJPdO4jvQR|a9P0faQNLaI)tGYUTW=xf3O26H=`)J@YTT= z0Jt|sC7a%WXn$a!@y3SHeN%@8l=1ww*KhLFX%u`8$!K{k62CX{qj36B;3&ZGa^0Mr zZ;yYBg2ccGGF*KF10VF(+Il92+;E{}yZ9GK^1a^fiMs$>j*z22t5;5i^w9ZS$A6 zJv_d3Gqt5GK^sSzXFtqHp8l7|?Z2&4g zQ`jW{vdv5+AKCu?=i4q1@QOM=Dcr;Rns1?VHVf{JVTWv$wA|}iY?^D9wZi(iHR+pf ztE{-~$TGqLpCt!XG@om}zb>Bgd)ps}2gQa$rT}nFYO+1!azXuZLDc*wFhhkbkWcHe zYQCuhHB3y`TORBLjZZ+JZ#+1Wu?(Ecy!Y#ACC-N02P$z!?g|@c%}a7#qn#QQR}Y8F zZQk_$GEgPK`sSzhdoYzGB;Jg;RZKgRr&msPSA-pk4k7oL0qvR-;r#O+BADY;W{K*i zr5o73*A{68T0%lX))N~o620yz_OVQl?R96`j(F3*8h%N*Qrc(5!_Q&aPZ{%M?pc*+wG0T!Kq5mEf*!_Fme_ zvzDybPo>pIoW#^f(rcn;c0u(?F!Ldi{poJo1w^Xngq^rvJHKj1i64WVaGa}%E};_v zlUsZVwg@DY0fmt^O%EOv*}4%B;j-RuNIw|&(1_v!iD#hB&af z&-}@Jc(k@sAM3!|>yoJ2Ovq(9EO?@Cr7d(A*YuEGY#E=P_R{DJFiPE5z|SL#l)eWbvt95P?QA^<%P zj&bGp&Cb?l7NW?J$lVz&=L;;N$U7V?N^aZ-BK7+X$uUUW{={)>(XK630o~&^XY3#s z*b&C?b1k*D{UZ6xejsc#GwG$ZsAOgHRYVgLo7h?b^7xcuJTmxDsMi><^d5TGlxQB}>I$o5hTRCr(M0#d0Tc>lVpuNMEK-(GTkXvYeCbD< zEvO^55S?=F2PJ)|Vx}?PM*nuRWgblP`E)$txlD0NM>!96wv1LGN>Wj+1_tgp2=J(g zNV1>0=MA18Z@VT-bk&;WV`?%EqF!(P0}SXr!90rtu`0Isz#1MP!%3`idQ(R^^Y=0^ z$by-W+4EdMS$=nxLiox%DW)jYGwWTO01x}`%SKa3r>lbI#)v5ar0^_Il9Sy>ZaCJ| zKLt#rpm4%vY;oRefJg8z2-nw|Pi;X>q|u3G%Tr{(Ms(dofwwGWgh&mx&C3sU>Akua z+ktrlyqH%6-yL9XQ7zs!=F5z#HCrCm^uUXZn7ib`c852_@aYfibNID+eA!UqMc-C^ z+0mu(d|HRdjJ+&d>4Iw&O2Qd@hY za%B<^rJt2S5tWt|R-ek`A{4-V*H&2LA}Wh_Fx1xKsud3aG$TR+i?@3hn*+wEs}t0> zMCP!IB0r>#X~9#aW&O8!X6uDwYZqj%o)SbRgUku_MWvss3yNvS_71z3)AH@|u`%+R zS!w={R+q!q7-cK-UEJ6Q*M>uv=O@di%Eo=G{}OTjcy}Lk-eg=eHjvj4Se=fTTVaG( z<@bft@@ybiCeJ?M4Al-_$_jT~>o_ zQrcldEF$>)ANDSNFYe43Tsm|T>wYPJ57TL;Q(IvXA-*Yee zH3%m;w?5Z@S6N&+m%G$StK1*pqJ8XNDX-LU@#LyYGh$U%OO${2bGi*sq9rIJu>5aB zNp_@>o?A-P*1pxrzOmavF>z?k6d4cL`W7%0{s5QT-0RbWS#G(qnk-M@_paR?L=EoE zKzx*Ul1x7GbNz!FP8CcG*8T7!mKHxzodN3p*0V=}8|&LFPanx6Pu(TI(jbFGkQ5@l zwdCjBle6vE*pD{*w?tutbn7MG+`zD@t~K&vBO)S<hd^aZIv7-y^7~gen ztS#tQq+RL=UF0=pklXSMDPDMx8?}@b5TyiqD%tXuicDu@qL6A(%cVFyyppA|l5L226ElGn*BmuwLJPxJ%k&7H~ofo_OwH)79X;WmSPvL^@i|L~U>~Pb!%?GTZ^8+eN z{NcZUmFQOQ7$o-B6K3M+H-)GTTBHLp7&ePjl=hoE-?(^)%vIR_(UFYU@5;*)8Hl60 z5Wd*B)QD5g2%YZ$w!{=1)A{b9AFO)j*EqVvVXE#~(!DzvF%IIE*Sn$$bXyr{nXkhu zbd|MINTTL@0_;dPEw1HWEOzY+WdUpPwI`sDFF?Ljit%;pX064P^?7WZJcFJ9;s^f~MJHxXDsV#!k231mq z8fio0O|;Cxm4jFzm)^BGyEW70$>Q`Fg|TBj!HkLvAr02dP&+MN?;G=d;o&ReRjY2@ zLxVesG5d)z2mXEC@+A^2W3|GCP3kGfcfHde|DM}gP{*HB#~*y^?jym`7A=7`m+gg*e4I!^=6-Eifq0KUk_#FNwN?erNZTADh%x-UrWCSom?h$ffnD68!uu zx}|kxiB|PJ(suSRU&!=_pdVM+e~OAQ>j~lU^bd!rD;gyBNDS)$am!3J`d!siUTji# zxynqvkizVjvqknbTULG^sIoBI{jd)0DTLp_gcXou*~*VNl>swptOZweZAl|p!%X;fSX;m?sDb|J$ zU+yaT+`v17u^23^g8Sp?tB0xAHAq)O?4$Tz9iy}hOEaM0=Jx$n%i2oST7XX9-bhw1 zK2(=VY$o`QWoJ=r&JD-WKL}A*&XU+;F)U6ZKW1jfAUk&A$LWA58(-#XL2RfVg-brB z@EQJgCV9To%`vJ)METd965WO!GrR&byh{JR?r;#mx`!l6Qbb=8tH4>l4VrVU7}dvH z&B~sUGEkZ`QKHLJ?Ok*?)f0Z#6MX-Dm5j5AD)@DAjEnqZJw8G61md$^)d_@$v}`_n zu)nHC*LLRbRldZl7!fvN%od`-mKk1-nbaMF-1SE(?G6^}w^p<`JA=FHeCN@Xuua_; z6p%n-@}~KBx_BogSPja{*E7L}1_h~~wMh4_W7s-sxKem^xv>Ynm50OQ?xQu$?Ly}g zBG0}B4i?r4es-utSBGyV6*EX21p|zuXMzEqC0(7k6s1r47*i1J(VHIl^Dn-@A6FHA z-WGaGpf@`nxb2LY`)0hZ0EimF@7_uG4i3sE|F`|Ib-d?F(Tn86vO3G{f<`kTs@bg$ z*&RNGt*b2ZZL_Zh^; zLU7n;>3#biLVEVt*$cGY$;r!z~V=d@7Ertj;fJZF_XlDHw}m%82Ev%c>jQuvL3A1%rc^?(`g|1W1rpB1F4E{Ri~*PWrX%crmqm;$1L zkQ&~1_w3th?3T*46-J%Sxu0?L*+W#O5OoyVrL;(`5g4}74_qnA!8};Hd+*wB>Y6p+ z)GGgp_7>XDU1{JV^dKQ%aWt!ILAE`cd3aba6=c zaBbF*-Hi9?+?aWl6(F3uZlee09N~-*Qe>Ow2~QclR%b>Rv_4`%S0M!P zl{p&cD{*zVkPvIwpyjT<81nu0c={}XHv+Q;>Y*}I&V#(}IAShG30q~R$CCG+Y6(&5 zc{>bfHJtqYm1TxVdWg?sn7yL^#Ft0XZU}PzXmf7jzv3*!T%2HO@9KQ9Th+=uHcG!> zkz&zRuEK1TM3DLFPqEd@KRzC^!iSDuECO@|Qml%Esl?GI7g=-u={C>0j^Dtq=Uep9{4kc+CMNR70{73 zKc#)*p<7f@)AC;;sgG&p!_ozNZ;hk{{=B0w`t06MQ6ZT@`HS^1;lGgyA7Q{Fhi~jL z_;Dn52f)mOIzclHB1NukCv5!MObhnia<=jQxX-H4DcXOk8x)33i&Q959;ru)-;7LH zhs4}*gahk7-#9TEbp-0y*eGjSbfJv;2;>rLLf9xk#`upmpWesQCpyafP6FskUZ3rt zJXQ7U4eW$RanJsMn(37ASAfGV*$0Ym53 zJ_rk3pK0eC$H*9^`F4x#XJ=#Ner`<*8q4-hLW93ZL4xs2R3l^<3FX7w+K*0Od^Z9x zBZj;~{Z(<6WPo-D{(SRjgeP$okl=nId@W$|-^o1Ky=rVxf19{tb zlL1468!n>R^1!B@B^8cOkr{B~j{n^FS6%xWYi-r8J5%toy~3(ON1F$bZ(XYB<(d4MB4Q?bv zQX|s&#ONqATr$~NG(QJPYhKCc$i`0(PBsvB{+__8v;y_vaVuD*a^^vPt&OciJwdqw z@+W8%;*oK`O}?)^bQ;vr+P6ic6Lw`llr8j)Yun3gt^2DpVrxe1yk4r^b33{Lvs?ty z1!%jj$m&xv3%cRU^r_IL{|(*^z_Yi+J-HL3Ap z-;&u%#M-pL&pZ(cfu%1IZyi4;%BMVl7Br8JHyVgr0|G$SO;aRto#TCZ4|cqBaRhb-+(MLvGr-x?D(AcEb(8XUZ=U zmhWP+y3W+mzd(Zd3NCqJo9|Y0izC^smzfBP3JFu#4>kdy#l9eP29i>w$l|{bH>LQ2 zhjkndQhPKu3T1*u=(<6yv}F8FBj9u*7(%B@mj3eGKhe-MKd9|hA9FxpO|GzMIbMrD z8%eRj6vc^6%*4|dhZ`9;57-{VM)d-I-1)3GOmXZ*Mm^!~=_6D_m6f-7rNak9C*ACz zQHE1_;c*l?sN=J(O#PqPU(S>Dhd2b*k7x!$3rTjA(~-U2!ND>55F2xc994hHH!8F> zrS}Lns^WuPK5g<-t7gV-t%F=`IJvB}Q|zTkZr`z%^5TF|Zex7{jrcp|T`2lX_XVfhkS1&WIE(tFJuX z=T*bO_Nt_+o=|rBC{J{Co<&vSYRL zNd&(uXL0)R&rQp}>ICxd87j+9No=`=F5`r{BNmP6`(A zup;~CBHo+hO?M?gqT)DAACaC0{^2y97`c)P`hQ`t6xisVH^ioKOqK23dj<+F&HE@W z@8Zxh?!7qCe7w{_xEz_9s@qf&!v7&Tu_L(Gr%9EwnaC6&Q~Sh)O|Mw$WmSq`qaMMb zg!J^G;gw9W(Ck%a_$qZ>52(!8uvwvsrMCQcP0j7?7SCfGkr}oBTI>yGw9%?o{cHe) z%Fw`eS*Vj!0yC*$!JnDgD23-%WvBMZ2q8k)B}H6hGag+>RA5j zISSXxa@E?hpzc%(GB1RqaxhSwl;R6p=tm}r1YA&jL?EP~vExWUgmv-sEl~bUR#=SK z@Rqz{nMB7Z#$Z!YB&nEIQNB-q|Cz=fL(CI){y!AqVzXO!wgMqv`jg9gw9op~FU2*}u+e21z#3@+@JD+b{Gms{r zn-9x}7eP%Q*_i#Vf)Ef>@O4#TkMX+9wf7_?^Mv1?F%Vv1U}&L znJ2qoZp$Ay!+;vtsq%Ze!gj*3lxRoxpG-aQ!SVqOlMd{%1f{X_iK1IQL1yAOwPI~@ zG%NjG)haBmlUguQm){GUmRQTZqc+p&zy*G)W^NpV-K>3FvWTf4itG zTJ?M8;meqv#O6e$XeuXW`ihPI*X3V8Cdx>XDG79U%aJ+FV{T2s{D3Bq<-R6{0~?&R zvZUDiYbQ4r2OtI_i~mF46{NK8R14pdIx zEEzvjTW#YRK65X<7LXl4;KP52-2VY2UQwaoI~_=nUgla9oPo^LuiNj?WvDg79+N+Z z?FUTvf8Wzn>=N(N$D?}(7NAp8MNz4M3OJd*!sDJem97D76VAQ~BAA(WOU3Jdn%8o|KN&hWJr z0&(~Rv6=sWc)SL}A9O`Z#=eSwCKmQ962yqGsYQ@%0&sy)FWcj5407cCf8HilL`Xv~ zwbv={ck{m83?xoVBWbQgc2GX}U;It|Gr3@=n1Z9fIoCPp1&S=ijO(eil+-h!7`ek! z5|;tL-Ta4Kl-Hdh{eg*&!(47DEIG(@^0-RrB4hzoPLiA`)rivrL~exHtK)!Jx60wxnjKMp2c&)f@+ z1yUEFL);;CN&8;D7Q6OK1@w(owYG8+5jI)j^7?rg% z#5sli;?e$E4V=?K3-8+$fJ8Y!1gb(fNf{6oM%NR9;^`g#zTyNIA=!2_M3U{5 z&vvVCqR`+$E2ZKfRYTeXL2ZUp@Q}CGLfY$B!yy|-=l;L%R_b{*97=Ca1Nul|_#t+S z5%1FiFiyVE*_NP|NZV&f@;Bv!q};|2=|@oF)d%3=fct1R0XD!->?9hbi&N8AXR}}_-UwGfUxaiTwmJxC3Tkgq8PF72 z>0wrYd9TtyZmxO&J*huXl0RCio<1@v|Ld_1UVV zd(jv%5VZ`5ga_;X=JfNdE-X`2M`hsy=iAF4x1|x9<>U88jHLQ~x9o(SUAILT={iP) z2H96G!hNei!3g$9x}M+@PyYzi9Z;jQKuqL)ob?6jPI&a%c2S7TP_n3d6M0su5-PPo z-A9U5%E3Uub;Aw;);#v-4JXl70bv5U>vvMx2QA$ViyyC(l?dfMhp@a*qFQ*ft#)Rz zz2Ge3>->N#e&f>kH}Kv>=xS|XO2BW2sGos-zX{}HHCM_h-O)0NpUa19RI3gADZcfXMQ(mpQO4 zbN{ldf0)*PIo21$DgV8eF5^>TBHFD z5`^pjdA;h6jf$rEP}()583{-Ua)fjB2WOz!<@ZRo2GzZ@JLT15F*DlTLk?SrG3@_E zwR6|AQ`!Sht<%b)I(FMFh zME+gy+aS@ap0EZqR}~j_S?ndr)3S;DHu!bb@FTax73yU zHG{+6TCteA^;fp7Xo{xT7)dk%j1s}v5Qr*S6=E;Nk}sOu_&tb1Bjr}m`{Ms1 z*+C5?uOxF}Gg^UQd@~#hfonr5YKKrQuCd|bI zh+D*g#vK}`bMaU*`OB8Za^QBCthDCAE>ps{-mMVG|4v|n{s$^g`$8P1NK;L2Gsj9WG^>P|;4s%d&RG#t? zXCvAIGyA2>ZR3A>0o(_P;B()AJwwfV|8=y#_ZBo|J`*+)saPS=`^ObFXz1_A$KMOO zckiB6RanLMd^>8N@inNQ9ion^k`CO$WWm5(Fs0*M*aJ@IvF?430$i4QBA_MV5$5jH?q3Sh$SDH*!+)=#;E*57IVM%K~tLXU$E|Wu$>4SBU?SF0p$Bei4jZJ{(Akth+-g(H>lr4`@fT;v{%Y9o`{8x>UQF*R%Y45 zr`eHQlzq-)Pn|a}5=)W#*dOj?XX0&)A<&+xq^vl)yCBsVHa6^1RNCJ(F2&cN6H^r| zTcp(lA_oHI0CjFzoWvGE&cg?;PJ;5H#d!phzG(81Q+xi83N9CR0LrbMGnGC93{@;-^}v@68Xd4+%yeM^T(wfbhA=iG3_sI#v98O!&MOYd1INZ&n~2mMzNBO!PE z0#8h>kKPpq)Rwb@gE|5VqR@y|LXkkA@N_(llmvdgt0gRiKVa1?75PO^TA^k{rV8ZEBU5OX>w+Z&i6 zS~hanGzSiE%>JgU*x}Or_Se0>6#Ml)-Kh`;xpS3v9V#pJ3D1TXb?ufCL@O6ABf~+R z+vsFA*4-+&tX760843cS0Y8_YR6ZySCv>?o4pfY2C``U>orO&FJo22Wa;AaO(g?J` z+Jfu9jResV3J?ApyYlpgip3`U_viX46D%a)`>ih0#}hAl-uYp83-eFzvfj87Rq*-V zqu~k~8z%-1u=!HBaga;ze6xn?uj?#wgWgR)FsK~R+Syh^nmj!<1KRRFi&a@C+4v_z zn=7TTYCI#>jP%0B8kGL>5qrVi;VLe{1XU}KDr5g1{++({dYU0aCtn<%IodtI}$ z6$X3(RNUgc00VqgB?9gE{M+pn=G(DzBH=S4K|SNPSwQGfd*`5yr!pf#AWUvbh2O_1 zo2U$kp+WL*+;L)9FO#H2ir0=p+IIkxB*6)4R2&B!ArgJU17FucCweDsI;8{c;)Na1 z&WEah1}vpHBsdn`jyrhaw4xvy*ujdFN~kq(W1#`Uc`*J2G~n_s!)aJ~)D@)j7j@(u z!$%Gq1w|+wp;!|anG~xc*9T^~Rv#x6QR)*=!zx8;>=nSR-XY(=3T@ADM`6b|VCW^? zGLm1%bK14m3*8CA^(2R~Um2uFP}A8wux^%VGY1eA~6 z6bpIMIqBa$ga|T@00(`*=$SGTC(T{~o`4fIQ%3PZBzn`Q%eyxD@M^^j)Y|Zie?8P|Ll+ew@(X$66q~QetmC9udUOQv%(iHq+`93>=OJt5i#HbDTAfb$IK9Ovu&#%R z@jP(|4LeY?XSa+;=QALln-qkv!k`uhsZ4}vftG!QinZ^+TnQoY;>_6Kn^K@j5Y8p6 zAM<-j8^;7kRgPCYks>5p=d>9?<7BqPr7}3CTUZ7~YWWz%q|9=J;e^^AYVRS`YpEX3 zBsW8t-9vjPwr~K*`pb;W(8><}<1#Bc#jCXB_|G$fo>?V))R zTELfsLPA2$3m+D#PX4?I-N*a3-L$T}D#YD6IscTu--}lyNmpsZrBXPr0F^E{8fV+) zT4dc^+X^#nwfp_iZlZ|DFoiFkkZ`K=_K|;lIEX}_^mEi|FCj3u>im?0uOC4UC#a4= z7=x~Cvmj_ue$USvJW)VU#$(c_na~0gnn!@|p7^jolwJl$$xFM5`x^%3{yD^_3VpP~ zaB2c>-xvAOx{o&#&MoJ|vAS%N{XM(j^e-Hc`Tt6~@^~oQ@BN5S){v#hQqtRN&7{E~ zB9)N7uicDfo$O&OL$uIhON+`j$yUT<8_O6(j8eo@6T^&BO$^35BV+m9)9;V|=<|7= z=U&b^*E#n+=epEC{id`4a{+8k;UeM^1o=4vu2Ti&?aBuyz~8|F`JKAl&YS&3fXNDY zW3|s(tSnaB@m9s1hCR1H=Or+fa_tVN{~S;`flE@eIso0gbYwC zLgAT&C!pi>E*$)4Bm}g~0_VBZ_^y7zEmz+H)L5_uKDnL0HAeuPoYDg$9{_= zM`Xe+PZdT8w~vQ0F>h6b|4mdDg<(A%~KtcDgItfKThXwa>92YtKgzS?f! z?D_CaNY<^Z-`vhZZjS-IlJ^n|LfbU$*Pq95zOIGvve8A*i~s)lMSR!KSip6-CYa<3 zXvQW|5Ek&}FEx%fNTj#aR$<&C5-J6ys=l&IZ##i~OF<-jpgk zz7Ru;tkKNR%OmYOnrr$~j{*UiPolVr$zPJs0LK7yBZ43NAQ}H+txkBBVKBx#oIe&I zM{v3oeHqe#OZQzz{Pst%58CoAFRu5&vq}JU%3d8>B>~=*Z67yjce+jP`$5ol4mI$^ zhyIF#fYg@A7+fho6wd_WAHkT!M8vO4cQwbPx23%Jxd?K_7YTcygcV{`k@@i3iaQ|sGb1MMnWXZUkL&4 zNr}HfqyTD2Yd;UkKx;3*K=3GBJq z@7=BG{l4_)iRMzgNA+TOmL_kDMx-viNjfM#IoCM{#ip3*t2~xGD^e~ zuGuW~P@gSAf6{RwD&>YDmn};TGqMvEu^0{tP!xU(YH8_|N6$=udq3<={!*&jd#Y!6 zYZqN$LaHF@%ZzUS5l@`}+s&3#X+LuuMNA<1*`P|%2?*75v37NFX3h0;=nc-6G~HPGw~r4w{!q^I5IXHeVUJ7?9w#@R>it`*L)h z!|-VBX4Tw_xWq}ghlS9|ci;Y_^TPG~1SXVjNc$1Tul(UUJ131uW0Y+J+P74{sX)4i zZd{OX86`Akt<2P!sWO!fgBF5WD|ri%P>~BY=1BerbJFIM^3&6;o3N*>A?q3b?9L3U z5E?$^scj*ayGq!+@L0u^6QQ&t-HM(6TdV0HJ3hSa?EE&-yyLwJ_V|rhsQGF%D$Yrc zFxfl381rDg6?-oeA8GW^HYL!+4_2FuN{gviv- zDv`5!ohp)Lgd4P&C5S`Wr|?njiz8mKN3-^m{3Q9)aX3pk(5-(eV`#w9kz#!}`2q!Y zni``~M-=r@*k>_B#fo0FQH7-by`kpseuZaFK;b(!c;)CrPb`kdiKLi5z`SI)I^22o z%7Q$3+u&V@k?$voEEj6y(Pp6(TUo2Stu-BEiMI^S6fqfd?^x7t7W_Fzp~-7K7k#Qe z&b%{|4}Aw7tE^nz_~zNl+vmG}IZ*~0%U3LU=~5MbqRB-uiF>qLzb~M^doJS&ju867 zg(n0Iw(k=>>pSU@Vu_!7fInyEx4GoWi3zQ#0n^NUXv{DvdktMXgbk-^!zK6+7Z^2D z?#(p#nptkPr%Hzi1T=a$Szgsc%i5j!EikqpJRU|i>pcXN-^dXRSmse|k9n6VgxDpq zh$bhP9kbJK^HvvnyEJwUp*g&ofs)If?CVlaAr`akEs%k6>}2s(4dT6c%r6UDu#;M4 z0w}-OcJ2Nnv`no&2A&W)btEesG!e041)!$r_gTcgv1AZ**Kj$vLyi~ z+4dNnr!Af9j~@3TM;_>7BXjPl*@GGniw=+^)GW7{@FO7Jufb>|s;tsbqIrkc$Q z>QPJo;=LarfEeXq>}V~@mDohn@mY`zQZrrA*nvNB9qS|q;;x!4nWFV*Ac)%Ylf9gz z2iP{x*&nPkaUjef4Gp8H^;fkWO1%hvowlV9W<6FNe}b zk9Ip2M8Oyc88Ai!Y~w;G&E?lp%dHtJvn99i6WK;^reBNKAO)8B@UxD**cnXt1*80ti#ru;&P~Ga^o3moW!m^E9cm6l zHx9fITU5VMV!;Ep%+-tsu(n8+KdQn}`}+-{!G(e8&!pLA#4d&MY+}*v3MJ|D z1}A~ft~pD87YH%hyVp2=+F&np%xV9(PcZuovyJWiy}e{5ioJTC{BbJC#Ovg!_$|YQ zbEE$qPSV*HXMa;HfqZ%3E#Soe*`n+)YGW6q%}YYfx6yes8pSM=z5mEsO?0nc1Bmep zqy6}}wRUIv%)lFc3O3aZ!s11vUxf65vh-5hr7iv?p=p+a}J9j_KY|W3$;! z?RLBy(h_(caqzn^W}@_^o{aHaqWsEH~BA0B-4?>vd>7 zhf5!L6X<>t-fht^T~@;)Mc(+Y8){nNZ~`$Vut+J{OvzS zMpy>7iBS>iX?|=V1-L@oB|5hEW8n}y3uF0}OG17XIP))R`FHBv=1`DKv-a%iVu_!d zm2B+WrFBO}FbO3)I(Uo!nv%NEzdSqg$6C`>(@@$O3Jm0^zLJ;wZZ-`0iCQiq9z`nq z=N;T*zb){H{e5kDv0(km^v}t)BOx=q^~N!jmh%Kc6i8=2iiW2-#ELxX*HmA&zNvCc zNi&UInt71KJ@5Q^1hez1Yfcdw&suwl^R%aY{mu!W`mW3z8%zf?NLh~!hz0SfXdkWjr`|}p9AxJv>j~#SQA-f-{VWI5 zF>cZp*EG!xm;*A;N!!URkh7_c%~kNG6v|8Zdi?B^6mOTcdR6!)dS6471A#Ux!B&d! z-1D5*&cHgFDQtLEvDQM;16otkm-v1?e_i_8!yB>4K;re*6WRDFT_+mBNVY&a8y9q% z)VC?=OCxYR4k%7ML3i8&cd%z^n(~%3LgP?n@^0S;m6hEQ3H@1Vi;vIW_$0EE3p2b1Dt9x&i)Vjn4 zBAfZ9A_d2<$GhiZol%cG?hexrB%J-J{DpN?EXb7)Zxd&ix4om!J=5nz>_`yoThXXstB#4{UM29sU5}4)C%-yz9Ft1=z?Y# zz4;uZd5RiyUM}_4%;8hI$QKWd+9*3~qlT!v@Wm{COhjx)!|lm0>?zi9aFCCkb%V$W zCL5DpAPC#pnl~DyeEm@T}V$)swY`Qe=5w1@~KZ`=WX z35})9L6@|ABxI^S->GNZj!yjQLr%E1d%zNBOUI&VnDAe1`(_XI51S;(PA?RKbWK9_ z!|cBo&{%SKHjnyhJZzS2y0Q`g*zYR#?*4aZJNWubr)M0GkKD50W091>bV6w8 zp6(WO&(2JDd^U)LPL<08r)dO@I*WT*Ba5aUFVyN-uo{03Z8o<@((8eFM|1`1N5{G3 zIGV8*Yh`p(Wt^?b2&k_uzdZHedTUKMUD{xLszc69ID&;+apo5W%~5n7NhNUHeu`IA zC^fSEPlGQh_fDA6-fM`O0c`l->vs0+O5Q%+ZyriMJvE6mHyWk<<4Rjchv#vf{$VR{ zNkVXYPB{|xPw%0s&+jm6dXWU=)c4NnNy6X+Ib`z*`y#7Q|wToKY^_{cSt$W_=IA-1s9$Lvcl239=eb8c#kLA%aT-v z=TX$g*@=p#{PJ2n5K;;k!O3iFY?oz@p>Fd6vjg`Gvtwf28AOn*|0G^{xb;7UhjENH ze(r!At@{sJ{JS5tCB|Yc6LcowquaxQSLT-8`Y0!zz5m7q;?r*ZneFS#EgPSe)(79M zoDh}gu#Hz$UG#!J&hbz#RS$2?9J^#HKIQQOWG-%x?aoq@LhJk=$hG$1N9&$2dN=d$ zG7Lzo!|olU^EH)~k1V|N=iZdr3y>p!SnZb53H|*WcuPxE7(TE!P+&`ceVX-*jRk(w zo~^qrF~+o78IIUCPwAND(aS@Xg(EhWqOV?56z1hEGe2|NZH@jS@?JwKcf*<96q82O zjhis%$6}m;y3h+&Lbzlt>Ia>bMJtPEhaRifwlig!h+p3@o_{%9___Oh)cE*5a-?2L zu-oP|C-&P$MRLCIeDqB-T>s6+Nk0&^NfhPM zX{TBvmCFw8WUXygl7ygjt=X|6|0i%}*y!7&ymHdev*u(zvd#yiPsf`CqGG3;XtQ>6 z)>w9^&k56bvLdKTgN3(G{6VKWWH{An(}KrhtGEJ5dNLy_ISI>I>-ZZWYN3HM{ZmrR z_n_8#q*G57^Jfxn3(<< z$QBMVjRMWhv^F^$&OL>kse(TZPM`I~3Bh<%3^eB{`dWSp9I@-PZLCAom%d!gsB_bv z2{9wiX5Z$2q2_S!p(F1&yvLTFzO zqZ_C_q>nb8z+z(JacB8!kNDpnkHRPd?R>?e!O5{ibQTmMHT1+qVFQTA9t?RxHs;P%#<{p>gI;yejg?m7iWjbntlT zl)t7wu~Xem&3-WLn7yy*5EsK)9v!xk3YC6tX--n{eefZ`SAoZEEfFoS%NRnbruTGZ z6_uu=)=c8%;)`hC4;IPt=YmLuJ)iE0O)yF%p(yjSTd?M!$>VI2pZh^NmRRj$5vsAe zuweO$+`T95HC=EK(PY!!TBFluF}g%#V@)aMS(Dx4$$u+zs3hA@o>}+s@ zz*_OJ##q?W2yGk`|DX-EDTl6N+)PlpWyt0F{x-edLv$>yF+8s~&qrl$G`JH;H^|R} zz=}gex=al{sbQ5?9^N%l=tvz8VixFpGBZiiDZ?AK$@2j)4&Caz?yhEUM~-=vgWQVGPpKMj&w1l8+wndqpDD2%P^DP~a=u z5)0?lCShxj(Lo9$+vO!zy|=e6WA;FWQR0GAIS?dtlm9mYlyrOJkPh3aq zhBJ2)C%;pYcvp2PxvbcfG&nU_=A+E4_o$v(Ukfp+S6b?H6N-QJ`!WH~v~FIF;6AAk;tfb&mOso@v#P{hrKW%nNpv1Fauu)N;QF=N~3B6GL) zJ&_(smJsatyU*Z;$pM_{3D;e7%Vq~$o&7C+h~wuu;$DwAEgubTj+)_mqu^S6&+L>v z1!Sg^!Y9k>wChuFbX2vA{Is#BRKdaCAjH z{trA2ZZso6f{3}pgzv1$C|?*;Ys?G_&nOZ|m8l@Hn3QrpaH<|Ze*^US}*gl6so&OPaP=-k{F(yxIyODjkWF+AddDjtD|t{hM(iS_}rM5ZIDBazY* zbUxZa5itn?tMw@s`5&%`uL3k{zrqKSNc^NZ%2w2pPN!S@Q6EIMc-`r%8ACE(R(wNd zUx%{!cLq=j@qy9yCA#P8<$;j_G=qE3pJ+E4?e{3kp|bKGI`0iu4-}V}C9sJ)GqT_3 zJTr#gQ9;c=9%l2SgVETx8P#bVSjKK`7DR~gjRoz?6ONFW)gZpYs}C4!#TTXbC0pwC z1__)Cf8>w$#uJ<9Sa4kBY*?6^*Ze6lYg}G9s~U)m+qDZ#`*KAOKtDTD+1B&@(EdH0 z_23>1Y)M(E?_8SCE5I%CA29ywc^y@}@lc8U0b_SF+@4W9q{P2xK4;qul`%$_ThoJ8+&b1&HH3}(&D zf>>=U-_X?Q>OW1UZ?)D`3!!*>OOvFa^$h+-L0*;~^sUcx$*pShBTC7nBzTcyc<@q( zui(nJ*+}KBKSk;{lJjMZ1`zO&{P+7o_*U$G$Qw2gyxjn#!~ F{|6jiPDB6z diff --git a/docs/snapshotting/versioning.md b/docs/snapshotting/versioning.md index 1e34b172634..68a6ebdca78 100644 --- a/docs/snapshotting/versioning.md +++ b/docs/snapshotting/versioning.md @@ -3,13 +3,27 @@ This document describes how Firecracker persists its state across multiple versions, diving deep into the snapshot format, encoding, compatibility and limitations. ## Introduction -The design behind the snapshot implementation enables version tolerant save and restore across multiple Firecracker versions which we call a version space. For example, one can pause a microVM, save it to disk with Firecracker version **0.26.0** and later load it in Firecracker version **0.24.0**. It also works in reverse: Firecracker version **0.26.0** loads what **0.24.0** saves. +The design behind the snapshot implementation enables version tolerant save and restore across multiple Firecracker versions which we call a version space. For example, one can pause a microVM, save it to disk with Firecracker version **0.23.0** and later load it in Firecracker version **0.24.0**. It also works in reverse: Firecracker version **0.23.0** loads what **0.24.0** saves. -Below is an example graph showing backward and forward snapshot compatibility for a version space formally represented as `[v2; vn]`. This is the general picture, but keep in mind that future corner cases might remove edges from this graph. +Below is an example graph showing backward and forward snapshot compatibility. This is the general picture, but keep in mind that when adding new features some version translations would not be possible. ![Version graph]( ../images/version_graph.png?raw=true "Version graph") +A non-exhaustive list of how cross-version snapshot support can be used: + +Example scenario #1 - load snapshot from older version: +* Start Firecracker v0.23 → Boot microVM → *Workload starts* → Pause → CreateSnapshot(snap) → kill microVM +* Start Firecracker v0.24 → LoadSnapshot → Resume → *Workload continues* + +Example scenario #2 - load snapshot in older version: +* Start Firecracker v0.24 → Boot microVM → *Workload starts* → Pause → CreateSnapshot(snap, “0.23”) → kill microVM +* Start Firecracker v0.23 → LoadSnapshot(snap) → Resume → *Workload continues* + +Example scenario #3 - load snapshot in older version: +* Start Firecracker v0.24 → LoadSnapshot(older_snap) → Resume → *Workload continues* → Pause → CreateSnapshot(snap, “0.23”) → kill microVM +* Start Firecracker v0.23 → LoadSnapshot(snap) → Resume → *Workload continues* + ## Overview Firecracker persists the microVM state as 2 separate objects: @@ -19,15 +33,15 @@ Firecracker persists the microVM state as 2 separate objects: *The block devices attached to the microVM are not considered part of the state and need to be managed separately.* ### Guest memory -The guest memory file is agnostic to Firecracker's version tolerant save/restore implementation as its contents are fully owned by the guest OS. The microVM memory is by choice saved as a dump of each page or as a sparse file of guest modified pages only. +The guest memory file contains the microVM memory saved as a dump of all pages. + ### MicroVM state -The microVM state file stores the internal state of the VMM (device emulation, KVM and vCPUs) with 2 exceptions - serial emulation and vsock backend. +In the VM state file, Firecracker stores the internal state of the VMM (device emulation, KVM and vCPUs) with 2 exceptions - serial emulation and vsock backend. -While we continuously improve and extend Firecracker's features by adding new capabilities, devices or enhancements, the microVM state file may change both structurally and semantically with each new release. The state file includes versioning information and each Firecracker release implements distinct save/load logic for the supported version space. +While we continuously improve and extend Firecracker's features by adding new capabilities, devices or enhancements, the microVM state file may change both structurally and semantically with each new release. The state file includes versioning information and each Firecracker release implements distinct save/restore logic for the supported version space. ## MicroVM state file format -### A closer look A microVM state file is further split into four different fields: | Field | Bits| Description | @@ -40,18 +54,18 @@ A microVM state file is further split into four different fields: Note: the last 16 bits of `magic_id` encode the storage version which specifies the encoding used for the `version` and `state` fields. The current implementation sets this field to 1, which identifies it as a [Serde bincode](https://github.com/servo/bincode) compatible encoder/decoder. ### Version tolerant ser/de -Firecracker reads and writes the `state` blob of the snapshot by using per version, separate serialization and deserialization logic. This logic is mostly autogenerated by a Rust procedural macro based on `struct`, `enum` and `union` annotations. Basically, one can say that these structures support versioning. The versioning logic is generated by parsing a structure's history log (encoded using Rust annotations) and emitting Rust code. +Firecracker reads and writes the `state` blob of the snapshot by using per version, separate serialization and deserialization logic. This logic is mostly autogenerated by a Rust procedural macro based on `struct` and `enum` annotations. Basically, one can say that these structures support versioning. The versioning logic is generated by parsing a structure's history log (encoded using Rust annotations) and emitting Rust code. Versioned serialization and deserialization is divided into two translation layers: - - structural translator, + - field translator, - semantic translator. -The _structural translator_ implements the logic to convert between different versions of the same Rust POD structure: it can deserialize or serialize from source version to target. +The _field translator_ implements the logic to convert between different versions of the same Rust POD structure: it can deserialize or serialize from source version to target. The translation is done field by field - the common fields are copied from source to target, and the fields that are unique to the target are (de)serialized with their default values. -The _semantic translator_ works on top of the structural one and its only concern is translating the semantics of the serialized/deserialized fields. +The _semantic translator_ is only concerned with translating the semantics of the serialized/deserialized fields. -The _structural translator_ is generated automatically through a procedural macro, and the _semantic translation methods_ have to be annotated in the structure by the user. +The _field translator_ is generated automatically through a procedural macro, and the _semantic translation methods_ have to be annotated in the structure by the user. This block diagram illustrates the concept: @@ -59,22 +73,33 @@ This block diagram illustrates the concept: ../images/versionize.png?raw=true "Versionize layers") -## Bincode encoding -During research and prototyping we have considered multiple storage formats. The criteria used for comparing these were performance, size, Rust support, specification, versioning support, community popularity and tooling. Performance, size and Rust support are hard requirements while all others can be the subject of trade-offs. +## VM state encoding + +During research and prototyping we considered multiple storage formats. The criteria used for comparing these are: performance, size, rust support, specification, versioning support, community and tooling. Performance, size and Rust support are hard requirements while all others can be the subject of trade offs. +More info about this comparison can be found here (https://github.com/firecracker-microvm/firecracker/blob/9d427b33d989c3225d874210f6c2849465941dc0/docs/snapshotting/design.md). + +Key benefits of using *bincode*: -More info about this comparison can be found [here](https://github.com/firecracker-microvm/firecracker/blob/9d427b33d989c3225d874210f6c2849465941dc0/docs/snapshotting/design.md#we-also-looked-at-these-other-options). +* Minimal snapshot size overhead +* Minimal CPU overhead +* Simple implementation -Key benefits of using `bincode` versus everything else are: -- Minimal snapshot size overhead, -- Minimal CPU overhead, -- Simple implementation. +The current implementation relies on the Serde bincode encoder (https://github.com/servo/bincode). + +Versionize is compatible to Serde with bincode backend: structures serialized with versionize at a specific version can be deserialized with Serde. Also structures serialized with serde can be deserialized with versionize. ## Snapshot compatibility ### Host kernel -The minimum kernel version required by Firecracker snapshots is 4.14. Snapshots can be saved and restored on different kernel versions without issues. +The minimum kernel version required by Firecracker snapshots is 4.14. Snapshots can be saved and restored on the same kernel version without any issues. There might be issues when restoring snapshots created on different host kernel version even when using the same Firecracker version. + +SnapshotCreate and SnapshotLoad operations across different host kernels is considerted unstable in Firecracker as the saved KVM state might have different semantics on different kernels. + + ### Device model +The current Firecracker devices are backwards compatible up to the version that introduces them. Ideally this property would be kept over time, but there are situations when a new version of a device exposes new features to the guest that do not exist in an older version. In such cases restoring a snapshot at an older version becomes impossible without breaking the guest workload. + The microVM state file links some resources that are external to the snapshot: - tap devices by device name, - block devices by block file path, @@ -88,16 +113,18 @@ new Firecracker process. ### CPU model -Firecracker microVMs can run on any Intel x86_64 CPU models that support the hardware virtualizations extensions. Snapshots, however, are not compatible across all Intel x86_64 CPU models. They are only compatible if the CPU features exposed to the guest are an invariant when saving and restoring the snapshot. -The trivial scenario is creating and restoring snapshots on hosts that have the same CPU model. -To relax this limitation, Firecracker provides an API to select a CPU template. Firecracker CPU templates work by using CPUID masking that restricts the exposed features to a common denominator of multiple CPU models. +Firecracker micromVMs can run on Intel/AMD CPU models that support the hardware virtualizations extensions. Snapshots are not compatible across CPU architectures and even across CPU models of the same architecture. They are only compatible if the CPU features exposed to the guest are an invariant when saving and restoring the snapshot. The trivial scenario is creating and restoring snapshots on hosts that have the same CPU model. + +To make snapshots more portable across Intel CPUs Firecracker provides an API to select a CPU template which is only available for Intel - T2 and C3. These templates are mapped as close as possible to AWS T2/C3 instances in terms of CPU features. There are no templates available for AMD. Firecracker CPU templates mask CPUID to restrict the exposed features to a common denominator of multiple CPU models. + +It is important to note that guest workloads can still execute instructions that are being masked by CPUID and restoring and saving of such workloads will lead to undefined result. Firecracker retrieves the state of a discrete list MSRs from KVM, more specificically the MSRs corresponding to the guest exposed features. ## Implementation -To enable cross version snapshot save/load we have designed and built two crates: +To enable Firecracker cross version snapshots we have designed and built two crates: - [versionize](https://crates.io/crates/versionize) - defines the Versionize trait, implements serialization of primitive types and provides a helper class to map Firecracker versions to individual structure versions. -- [versionize_derive](https://crates.io/crates/versionize_derive) - exports a procedural macro that consumes structures, enums or unions and their annotations to produce an implementation of the `Versionize` trait +- [versionize_derive](https://crates.io/crates/versionize_derive) - exports a procedural macro that consumes structures and enums and their annotations to produce an implementation of the `Versionize` trait The microVM state file format is implemented in the [snapshot crate](../../src/snapshot/src/lib.rs) in the Firecracker repository. All Firecracker devices implement the [Persist](../../src/snapshot/src/persist.rs) trait which exposes an interface that enables creating from and saving to the microVM state. \ No newline at end of file From 89c9f74abd522c6842f3588bea9c6eec16ccb997 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 16 Oct 2020 16:20:24 +0300 Subject: [PATCH 3/9] Add snapshot known issues/limitations. Signed-off-by: Andrei Sandu --- docs/snapshotting/snapshot-support.md | 29 +++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/snapshotting/snapshot-support.md b/docs/snapshotting/snapshot-support.md index dd16306ae39..d5163f0ed97 100644 --- a/docs/snapshotting/snapshot-support.md +++ b/docs/snapshotting/snapshot-support.md @@ -9,6 +9,13 @@ guest workload at that particular point in time. ## Snapshotting in Firecracker +### Supported platforms + +The Firecracker snapshot feature is in [developer preview](docs/RELEASE_POLICY.md) +on all CPU micro-architectures listed in [README](../README.md#supported-platforms) +except ARM which is not supported. + +### Overview A Firecracker microVM snapshot can be used for loading it later in a different Firecracker process, and the original guest workload is being simply resumed. @@ -31,9 +38,8 @@ flexibility to our snapshotting support. This means that taking a snapshot resul in multiple files that are composing the full microVM snapshot: - the guest memory file, - the microVM state file, -- zero or more disk files (depending on how many the guest had; these are - **managed by the users**, which means they need to externally back up their - block devices backing files). +- zero or more disk files (depending on how many the guest had; these are +**managed by the users**). The design allows sharing of memory pages and read only disks between multiple microVMs. When loading a snapshot, instead of loading at resume time the full @@ -45,7 +51,22 @@ This has the advantage of very fast snapshot loading times, but comes with the c of having to keep the guest memory file around for the entire lifetime of the resumed microVM. -*Note*: Snapshotting is currently supported only on `x86_64` machines. +## Performance + +The Firecracker snapshot create/resume performance depends on the memory size, +vCPU count and emulated devices count. The Firecracker CI runs snapshots tests +on AWS **m5d.metal** instances and the baseline for snapshot resume latency +target is under **8ms** with 5ms p90 for a microvm with this specs: +2vCPU/512MB/1 block/1 net device. + +## Known issues and limitations + +- High snapshot latency on 5.4+ host kernels - +[#2129](https://github.com/firecracker-microvm/firecracker/issues/2129) +- Guest network connectivity is not guaranteed to be preserved after resume +- Restoring microVMs with vsock devices doesn't work. +- Poor entropy and replayable randomness when resuming multiple microvms which +deal with cryptographic secrets. Please see [Snapshot security and uniqueness](#snapshot-security-and-uniqueness) ## Firecracker Snapshotting characteristics From 3ac6670eb412ce969e49928b161f66819d589be2 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 19 Oct 2020 13:56:02 +0300 Subject: [PATCH 4/9] Update CPU support in README. Signed-off-by: Andrei Sandu --- README.md | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index afb15a28df0..bb7fa95e834 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,13 @@ in lightweight virtual machines, called microVMs, which combine the security and isolation properties provided by hardware virtualization technology with the speed and flexibility of containers. +## Overview The main component of Firecracker is a virtual machine monitor (VMM) that uses the Linux Kernel Virtual Machine (KVM) to create and run microVMs. Firecracker has a minimalist design. It excludes unnecessary devices and guest-facing functionality to reduce the memory footprint and attack surface area of each microVM. This improves security, decreases the startup time, and increases -hardware utilization. Firecracker currently supports Intel, AMD (preview) and -Arm (preview) CPUs. Firecracker has also been integrated in container runtimes, +hardware utilization. Firecracker has also been integrated in container runtimes, for example [Kata Containers](https://github.com/kata-containers/documentation/wiki/Initial-release-of-Kata-Containers-with-Firecracker-support) and [Weaveworks Ignite](https://github.com/weaveworks/ignite). @@ -120,6 +120,27 @@ The **API endpoint** can be used to: scenarios; applies a cgroup/namespace isolation barrier and then drops privileges. +## Supported platforms + +We continuously test Firecracker on machines with the following CPUs +micro-architectures: Intel Skylake, Intel Cascade Lake, AMD Zen2, ARM Cortex-A +aarch64. + +Firecracker is [generally available](docs/RELEASE_POLICY.md) on Intel x86_64 +and AMD x86_64 CPUs that offer hardware virtualization support, and that are +released starting with 2015. All production use cases should follow [these +production host setup instructions](docs/prod-host-setup.md). + +Firecracker is in [developer preview](docs/RELEASE_POLICY.md) (and not +supported for production workloads) on CPUs based on Arm Cortex-A aarch64 cores + that offer hardware virtualization support, and that are released starting +with 2015. + +Firecracker may work on other x86 and Arm 64-bit CPUs with support for hardware +virtualization, but any such platform is currently not supported and not fit +for production. If you want to run Firecracker on such platforms, please +[open a feature request](https://github.com/firecracker-microvm/firecracker/issues/new?assignees=&labels=&template=feature_request.md&title=%5BFeature+Request%5D+Title). + ## Performance Firecracker's performance characteristics are listed as part of the From 2d8e6636f4c4d7d10f35679a6c89d1da2cc4dac1 Mon Sep 17 00:00:00 2001 From: alindima Date: Tue, 20 Oct 2020 14:41:54 +0000 Subject: [PATCH 5/9] docs: update FAQ with supported CPUs. Signed-off-by: alindima --- FAQ.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/FAQ.md b/FAQ.md index 7e288f1eae5..457cb30ee88 100644 --- a/FAQ.md +++ b/FAQ.md @@ -36,10 +36,12 @@ to integrate with container ecosystems. ### What processors does Firecracker support? -The Firecracker VMM is built to be processor agnostic. Intel processors are -supported for production workloads. Support for AMD and Arm processors is in +The Firecracker VMM is built to be processor agnostic. Intel and AMD processors +are supported for production workloads. Support for Arm processors is in developer preview. +You can find more details [here](README.md#supported-platforms). + ### Can Firecracker be used within the container ecosystem? Yes. Firecracker is integrated with From 9ffb25b4072225e657e0504fdd9d7515fc6005df Mon Sep 17 00:00:00 2001 From: alindima Date: Tue, 27 Oct 2020 10:40:01 +0000 Subject: [PATCH 6/9] Release v0.23.0 Signed-off-by: alindima --- .mailmap | 4 ++++ CHANGELOG.md | 15 +++++++++---- CREDITS.md | 29 +++++++++++++++++++++++++ src/api_server/swagger/firecracker.yaml | 2 +- src/firecracker/Cargo.toml | 2 +- src/jailer/Cargo.toml | 2 +- 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/.mailmap b/.mailmap index e331bb6f6ec..6fbbce5a62b 100644 --- a/.mailmap +++ b/.mailmap @@ -11,3 +11,7 @@ Tamio-Vesa Nakajima Iulian Barbu Petre Eftime karthik nedunchezhiyan +Alin Dima +Andrei Sandu <54316454+sandreim@users.noreply.github.com> +Diana Popa +Alexandru Cihodaru diff --git a/CHANGELOG.md b/CHANGELOG.md index 845b8972d47..7485df97aa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ ### Added +- Added devtool test `-c|--cpuset-cpus` flag for cpus confinement when tests + run. +- Added devtool test `-m|--cpuset-mems` flag for memory confinement when tests + run. + +## [0.23.0] + +### Added + - Added metric for throttled block device events. - Added metrics for counting rate limiter throttling events. - Added metric for counting MAC address updates. @@ -30,10 +39,8 @@ - Added a new API call, `PUT /snapshot/load`, for loading a snapshot. - Added new jailer command line argument `--cgroup` which allow the user to specify the cgroups that are going to be set by the Jailer. -- Added devtool test `-c|--cpuset-cpus` flag for cpus confinement when tests - run. -- Added devtool test `-m|--cpuset-mems` flag for memory confinement when tests - run. +- Added full support for AMD CPUs (General Availability). More details + [here](README.md#supported-platforms). ### Fixed diff --git a/CREDITS.md b/CREDITS.md index 26b1fa84703..54b943bc29f 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -21,6 +21,7 @@ Contributors to the Firecracker repository: * Aaron Hill * Abhijeet Kasurde * Adrian Catangiu +* Ahmed Abouzied * Alakesh * Aleksa Sarai * Alex Chan @@ -28,6 +29,8 @@ Contributors to the Firecracker repository: * Alexandra Iordache * Alexandru Agache * Alexandru Branciog +* Alexandru Cihodaru +* Alin Dima * Andreea Florescu * Andrei Casu-Pop * Andrei Cipu @@ -38,21 +41,30 @@ Contributors to the Firecracker repository: * Atsushi Ishibashi * Aussie Schnore * Babis Chalios +* Begley Brothers Inc +* Benjamin Fry +* bin liu * Bob Potter * Bogdan Ionita +* Caleb Albers <7110138+CalebAlbers@users.noreply.github.com> +* Cam Mannett * chaos matrix * Chinmay Kousik * Chris Christensen +* Christian González * Christopher Diehl * cneira * Constantin Musca +* Damien Stanton * Dan Horobeanu * Dan Lemmond * Deepesh Pathak +* defunct * Denis Andrejew * Diana Popa * Dmitrii * Filippo Sironi +* Fraser Pringle * Gabe Jackson * Gabriel Ionescu * Garrett Squire @@ -70,7 +82,10 @@ Contributors to the Firecracker repository: * Iulian Barbu * James Turnbull * Javier Romero +* jonas serrano * Josh Abraham +* Josh McConnell +* Joshua Abraham * Julian Stecklina * karthik nedunchezhiyan * KarthikVelayutham @@ -81,7 +96,10 @@ Contributors to the Firecracker repository: * Liu Jiang * Lloyd * lloydmeta +* LOU Xun +* Luminita Voicu * maciejhirsz +* Malhar Vora * Manohar Castelino * Marc Brooker * Marco Vedovati @@ -89,14 +107,20 @@ Contributors to the Firecracker repository: * Massimiliano Torromeo * Matt Wilson * Mehrdad Arshad Rad +* Michael Saah +* Mihai Stan +* moricho * Nathan Hoang * Nathan Sizemore * Nicolas Mesa +* Nikolay Edigaryev * Noah Meyerhans +* not required * Peng Tao * Penny Zheng * Peter Hrvola * Petre Eftime +* Radu Iliescu * Radu Matei Lăcraru * Radu Weiss * Ram Sripracha @@ -111,16 +135,21 @@ Contributors to the Firecracker repository: * Serban Iorga * shakram02 * Shen Jiale +* Shion Yamashita * singwm * Sripracha +* Stefan Nita <32079871+stefannita01@users.noreply.github.com> * Tamio-Vesa Nakajima * tidux * Tim Bannister * Tim Deegan +* timvisee * Tyler Anton * Urvil Patel +* Wei Yang * Weixiao Huang * Wesley Norris +* wt-l00 * xibz * xiekeyang * YLyu diff --git a/src/api_server/swagger/firecracker.yaml b/src/api_server/swagger/firecracker.yaml index 4015415a61b..b284c304758 100644 --- a/src/api_server/swagger/firecracker.yaml +++ b/src/api_server/swagger/firecracker.yaml @@ -5,7 +5,7 @@ info: The API is accessible through HTTP calls on specific URLs carrying JSON modeled data. The transport medium is a Unix Domain Socket. - version: 0.22.0 + version: 0.23.0 termsOfService: "" contact: email: "compute-capsule@amazon.com" diff --git a/src/firecracker/Cargo.toml b/src/firecracker/Cargo.toml index c04c6b8d1ce..81efc5c3efc 100644 --- a/src/firecracker/Cargo.toml +++ b/src/firecracker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "firecracker" -version = "0.22.0" +version = "0.23.0" authors = ["Amazon Firecracker team "] edition = "2018" build = "../../build.rs" diff --git a/src/jailer/Cargo.toml b/src/jailer/Cargo.toml index 816bf8ceb9c..4f475ea0920 100644 --- a/src/jailer/Cargo.toml +++ b/src/jailer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jailer" -version = "0.22.0" +version = "0.23.0" authors = ["Amazon Firecracker team "] edition = "2018" build = "../../build.rs" From 67a37cc33f03fed6d0261dfa8d0c76249b8584f7 Mon Sep 17 00:00:00 2001 From: alindima Date: Tue, 27 Oct 2020 14:33:13 +0000 Subject: [PATCH 7/9] Update Cargo.lock for v0.23.0 Signed-off-by: alindima --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba4e5ffeca7..d12df8bdea8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -297,7 +297,7 @@ dependencies = [ [[package]] name = "firecracker" -version = "0.22.0" +version = "0.23.0" dependencies = [ "api_server 0.1.0", "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", @@ -343,7 +343,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "jailer" -version = "0.22.0" +version = "0.23.0" dependencies = [ "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.6 (registry+https://github.com/rust-lang/crates.io-index)", From 6ce93e9908433563b4b8b48f72e1528fdce8c6cf Mon Sep 17 00:00:00 2001 From: Iulian Barbu Date: Fri, 18 Sep 2020 22:42:22 +0300 Subject: [PATCH 8/9] serial: buffer limit regression test Signed-off-by: Iulian Barbu --- .../functional/test_serial_io.py | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/functional/test_serial_io.py b/tests/integration_tests/functional/test_serial_io.py index ba0388b98c0..6a6c2e76d1b 100644 --- a/tests/integration_tests/functional/test_serial_io.py +++ b/tests/integration_tests/functional/test_serial_io.py @@ -1,9 +1,15 @@ # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 """Tests scenario for the Firecracker serial console.""" + +import fcntl +import os +import termios import time + from framework.microvm import Serial from framework.state_machine import TestState +import framework.utils as utils class WaitLogin(TestState): # pylint: disable=too-few-public-methods @@ -68,11 +74,57 @@ def test_serial_console_login(test_microvm_with_ssh): serial = Serial(microvm) serial.open() - - # Set initial state - wait for 'login:' prompt current_state = WaitLogin("login:") while not isinstance(current_state, TestFinished): output_char = serial.rx_char() current_state = current_state.handle_input( serial, output_char) + + +def get_total_mem_size(pid): + """Get total memory usage for a process.""" + cmd = f"pmap {pid} | tail -n 1 | sed 's/^ //' | tr -s ' ' | cut -d' ' -f2" + rc, stdout, stderr = utils.run_cmd(cmd) + assert rc == 0 + assert stderr == "" + + return stdout + + +def send_bytes(tty, bytes_count, timeout=60): + """Send data to the terminal.""" + start = time.time() + for _ in range(bytes_count): + fcntl.ioctl(tty, termios.TIOCSTI, '\n') + current = time.time() + if current - start > timeout: + break + + +def test_serial_dos(test_microvm_with_ssh): + """Test serial console behavior under DoS.""" + microvm = test_microvm_with_ssh + microvm.jailer.daemonize = False + microvm.spawn() + microvm.memory_events_queue = None + + # Set up the microVM with 1 vCPU and a serial console. + microvm.basic_config(vcpu_count=1, + add_root_device=False, + boot_args='console=ttyS0 reboot=k panic=1 pci=off') + microvm.start() + + # Open an fd for firecracker process terminal. + tty_path = f"/proc/{microvm.jailer_clone_pid}/fd/0" + tty_fd = os.open(tty_path, os.O_RDWR) + + # Check if the total memory size changed. + before_size = get_total_mem_size(microvm.jailer_clone_pid) + send_bytes(tty_fd, 100000000, timeout=1) + after_size = get_total_mem_size(microvm.jailer_clone_pid) + assert before_size == after_size, "The memory size of the " \ + "Firecracker process " \ + "changed from {} to {}." \ + .format(before_size, + after_size) From 8c75077b84ee9ef31cb2122a124b7723485e7990 Mon Sep 17 00:00:00 2001 From: Iulian Barbu Date: Thu, 17 Sep 2020 12:16:38 +0300 Subject: [PATCH 9/9] serial: limit the in buffer capacity Signed-off-by: Iulian Barbu Suggested-by: Andreea Florescu --- src/devices/src/legacy/serial.rs | 381 +++++++++++++----- src/devices/tests/integration_tests.rs | 268 ++++++++++-- src/vmm/src/builder.rs | 16 +- .../integration_tests/build/test_coverage.py | 2 +- 4 files changed, 543 insertions(+), 124 deletions(-) diff --git a/src/devices/src/legacy/serial.rs b/src/devices/src/legacy/serial.rs index e3202cc9df0..cbff5a81276 100644 --- a/src/devices/src/legacy/serial.rs +++ b/src/devices/src/legacy/serial.rs @@ -7,16 +7,16 @@ use std::collections::VecDeque; use std::io; -use std::os::unix::io::AsRawFd; +use std::os::unix::io::{AsRawFd, RawFd}; use logger::{error, warn, Metric, METRICS}; -use polly::event_manager::{EventManager, Subscriber}; +use polly::event_manager::{EventManager, Pollable, Subscriber}; use utils::epoll::{EpollEvent, EventSet}; use utils::eventfd::EventFd; use crate::bus::BusDevice; -const LOOP_SIZE: usize = 0x40; +const FIFO_SIZE: usize = 64; const DATA: u8 = 0; const IER: u8 = 1; @@ -77,6 +77,7 @@ pub struct Serial { in_buffer: VecDeque, out: Option>, input: Option>, + buffer_ready_evt: Option, } impl Serial { @@ -84,6 +85,7 @@ impl Serial { interrupt_evt: EventFd, out: Option>, input: Option>, + buffer_ready_evt: Option, ) -> Serial { let interrupt_enable = match out { Some(_) => IER_RECV_BIT, @@ -102,6 +104,7 @@ impl Serial { in_buffer: VecDeque::new(), out, input, + buffer_ready_evt, } } @@ -110,18 +113,19 @@ impl Serial { interrupt_evt: EventFd, input: Box, out: Box, + buffer_ready_evt: Option, ) -> Serial { - Self::new(interrupt_evt, Some(out), Some(input)) + Self::new(interrupt_evt, Some(out), Some(input), buffer_ready_evt) } /// Constructs a Serial port ready for output but with no input. pub fn new_out(interrupt_evt: EventFd, out: Box) -> Serial { - Self::new(interrupt_evt, Some(out), None) + Self::new(interrupt_evt, Some(out), None, None) } /// Constructs a Serial port with no connected input or output. pub fn new_sink(interrupt_evt: EventFd) -> Serial { - Self::new(interrupt_evt, None, None) + Self::new(interrupt_evt, None, None, None) } /// Provides a reference to the interrupt event fd. @@ -189,7 +193,7 @@ impl Serial { } DATA => { if self.is_loop() { - if self.in_buffer.len() < LOOP_SIZE { + if self.in_buffer.len() < FIFO_SIZE { self.in_buffer.push_back(value); self.recv_data_interrupt()?; } @@ -219,11 +223,17 @@ impl Serial { DLAB_HIGH if self.is_dlab_set() => (self.baud_divisor >> 8) as u8, DATA => { self.del_intr_bit(IIR_RECV_BIT); - if self.in_buffer.len() <= 1 { + METRICS.uart.read_count.inc(); + let byte = self.in_buffer.pop_front().unwrap_or_default(); + + if self.in_buffer.is_empty() { self.line_status &= !LSR_DATA_BIT; + if self.signal_buffer_ready().is_err() { + error!("Could not signal that serial device buffer is ready."); + } } - METRICS.uart.read_count.inc(); - self.in_buffer.pop_front().unwrap_or_default() + + byte } IER => self.interrupt_enable, IIR => { @@ -240,29 +250,112 @@ impl Serial { } } + #[inline] + fn avail_buffer_capacity(&self) -> usize { + FIFO_SIZE.checked_sub(self.in_buffer.len()).unwrap_or_else(|| + panic!( + "Errored out due to serial device buffer size greater than the maximum expected size: {} > {}.", + self.in_buffer.len(), + FIFO_SIZE + ) + ) + } + fn recv_bytes(&mut self) -> io::Result { + let avail_cap = self.avail_buffer_capacity(); + if avail_cap == 0 { + return Err(io::Error::from_raw_os_error(libc::ENOBUFS)); + } + if let Some(input) = self.input.as_mut() { - let mut out = [0u8; 32]; - return input.read(&mut out).and_then(|count| { - if count > 0 { - self.raw_input(&out[..count])?; - Ok(count) - } else { - Ok(0) - } - }); + let mut out = vec![0u8; avail_cap]; + let count = input.read(&mut out)?; + if count > 0 { + self.raw_input(&out[..count])?; + } + + return Ok(count); } - Ok(0) + Err(io::Error::from_raw_os_error(libc::ENOTTY)) } fn raw_input(&mut self, data: &[u8]) -> io::Result<()> { + // Fail fast if the serial is serviced with more data than it can buffer. + if data.len() > self.avail_buffer_capacity() { + return Err(io::Error::from_raw_os_error(libc::ENOBUFS)); + } + if !self.is_loop() { self.in_buffer.extend(data); self.recv_data_interrupt()?; } Ok(()) } + + #[inline] + fn serial_input_fd(&self) -> RawFd { + self.input.as_ref().map_or(-1, |input| input.as_raw_fd()) + } + + #[inline] + fn buffer_ready_evt_fd(&self) -> RawFd { + self.buffer_ready_evt + .as_ref() + .map_or(-1, |buf_ready| buf_ready.as_raw_fd()) + } + + #[inline] + fn consume_buffer_ready_evt(&self) -> Result { + self.buffer_ready_evt + .as_ref() + .map_or(Ok(0), |buf_ready| Ok(buf_ready.read()?)) + } + + #[inline] + fn signal_buffer_ready(&self) -> Result<(), io::Error> { + self.buffer_ready_evt + .as_ref() + .map_or(Ok(()), |buf_ready| Ok(buf_ready.write(1)?)) + } + + fn handle_ewouldblock(&self, ev_mgr: &mut EventManager) { + let buffer_ready_fd = self.buffer_ready_evt_fd(); + let input_fd = self.serial_input_fd(); + if input_fd < 0 || buffer_ready_fd < 0 { + error!("Serial does not have a configured input source."); + return; + } + + if ev_mgr.subscriber(input_fd).is_err() { + match ev_mgr.subscriber(buffer_ready_fd) { + Ok(serial) => { + match ev_mgr.register( + input_fd, + EpollEvent::new(EventSet::IN, input_fd as u64), + serial.clone(), + ) { + // Bytes might had come on the unregistered stdin. Try to consume any. + Ok(_) => self.signal_buffer_ready().unwrap_or_else(|err| { + error!( + "Could not signal that serial device buffer is ready: {:?}", + err + ) + }), + Err(e) => { + error!( + "Could not register the serial input to the event manager: {:?}", + e + ); + } + } + } + Err(e) => { + error!("Could not get the serial device subscriber: {:?}", e); + } + } + } + } } impl BusDevice for Serial { @@ -290,22 +383,31 @@ impl BusDevice for Serial { impl Subscriber for Serial { /// Handle events on the serial input fd. fn process(&mut self, event: &EpollEvent, ev_mgr: &mut EventManager) { - let source = event.fd(); + #[inline] + fn unregister_source(ev_mgr: &mut EventManager, source: Pollable) { + match ev_mgr.unregister(source) { + Ok(_) => (), + Err(_) => error!("Could not unregister the source: {}", source), + } + } - // We expect to be interested only in serial input. - let interest_list = self.interest_list(); - if interest_list.len() != 1 { - warn!("Unexpected events/sources interest list."); + let input_fd = self.serial_input_fd(); + let buffer_ready_fd = self.buffer_ready_evt_fd(); + if input_fd < 0 || buffer_ready_fd < 0 { + error!("Serial does not have a configured input source."); return; } - // Safe to unwrap. Checked before if interest list has one element. - let supported_event = interest_list.first().unwrap(); - - // Check if the event source is the serial input. - if supported_event.fd() != source { - warn!("Unexpected event source: {}", source); - return; + if buffer_ready_fd == event.fd() { + match self.consume_buffer_ready_evt() { + Ok(_) => (), + Err(err) => { + error!("Detach serial device input source due to error in consuming the buffer ready event: {:?}", err); + unregister_source(ev_mgr, input_fd); + unregister_source(ev_mgr, buffer_ready_fd); + return; + } + } } // We expect to receive: `EventSet::IN`, `EventSet::HANG_UP` or @@ -313,22 +415,34 @@ impl Subscriber for Serial { // read from the serial input. match self.recv_bytes() { Ok(count) => { - // Check if the serial input have to be unregistered. - let event_set = event.event_set(); - let unregister_condition = - event_set.contains(EventSet::ERROR) | event_set.contains(EventSet::HANG_UP); - if count == 0 && unregister_condition { - // Unregister the serial input source. - match ev_mgr.unregister(supported_event.fd()) { - Ok(_) => warn!("Detached the serial input due to peer error/close."), - Err(e) => error!( - "Peer is unreachable. Could not detach the serial input: {:?}", - e - ), + // Handle EOF if the event came from the input source. + if input_fd == event.fd() && count == 0 { + unregister_source(ev_mgr, input_fd); + unregister_source(ev_mgr, buffer_ready_fd); + warn!("Detached the serial input due to peer close/error."); + } + } + Err(e) => { + match e.raw_os_error() { + Some(errno) if errno == libc::ENOBUFS => { + unregister_source(ev_mgr, input_fd); + } + Some(errno) if errno == libc::EWOULDBLOCK => { + self.handle_ewouldblock(ev_mgr); + } + Some(errno) if errno == libc::ENOTTY => { + error!("The serial device does not have the input source attached."); + unregister_source(ev_mgr, input_fd); + unregister_source(ev_mgr, buffer_ready_fd); + } + Some(_) | None => { + // Unknown error, detach the serial input source. + unregister_source(ev_mgr, input_fd); + unregister_source(ev_mgr, buffer_ready_fd); + warn!("Detached the serial input due to peer close/error."); } } } - Err(e) => error!("error while reading stdin: {:?}", e), } } @@ -336,7 +450,13 @@ impl Subscriber for Serial { /// If serial input is present, register the serial input FD as readable. fn interest_list(&self) -> Vec { match &self.input { - Some(input) => vec![EpollEvent::new(EventSet::IN, input.as_raw_fd() as u64)], + Some(input) => match self.buffer_ready_evt.as_ref() { + Some(buf_ready_evt) => vec![ + EpollEvent::new(EventSet::IN, input.as_raw_fd() as u64), + EpollEvent::new(EventSet::IN, buf_ready_evt.as_raw_fd() as u64), + ], + None => vec![], + }, None => vec![], } } @@ -361,6 +481,7 @@ mod tests { #[derive(Clone)] struct SharedBuffer { internal: Arc>, + loopback: bool, } impl SharedBuffer { @@ -371,15 +492,28 @@ mod tests { write_buf: Vec::new(), evfd: EventFd::new(libc::EFD_NONBLOCK).unwrap(), })), + loopback: false, } } + + fn set_loopback(&mut self, loopback: bool) { + self.loopback = loopback; + } } impl io::Write for SharedBuffer { fn write(&mut self, buf: &[u8]) -> io::Result { - self.internal.lock().unwrap().write_buf.write(buf) + if self.loopback { + self.internal.lock().unwrap().read_buf.write(buf) + } else { + self.internal.lock().unwrap().write_buf.write(buf) + } } fn flush(&mut self) -> io::Result<()> { - self.internal.lock().unwrap().write_buf.flush() + if self.loopback { + self.internal.lock().unwrap().read_buf.flush() + } else { + self.internal.lock().unwrap().write_buf.flush() + } } } impl io::Read for SharedBuffer { @@ -432,9 +566,10 @@ mod tests { intr_evt.try_clone().unwrap(), Box::new(serial_in_out.clone()), Box::new(serial_in_out), + Some(EventFd::new(libc::EFD_NONBLOCK).unwrap()), ); // Check that the interest list contains one event set. - assert_eq!(serial.interest_list().len(), 1); + assert_eq!(serial.interest_list().len(), 2); // Process an invalid event type does not panic. let invalid_event = EpollEvent::new(EventSet::OUT, intr_evt.as_raw_fd() as u64); @@ -445,6 +580,30 @@ mod tests { serial.process(&invalid_event, &mut event_manager); } + #[test] + fn test_event_handling_ewould_block() { + let mut event_manager = EventManager::new().unwrap(); + + let intr_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap(); + let serial_in_out = SharedBuffer::new(); + + let mut serial = Serial::new_in_out( + intr_evt.try_clone().unwrap(), + Box::new(serial_in_out.clone()), + Box::new(serial_in_out.clone()), + Some(EventFd::new(libc::EFD_NONBLOCK).unwrap()), + ); + + // Process a spurious event, which will result in EWOULDBLOCK and unregister the serial input. + let spurious_ev = EpollEvent::new(EventSet::IN, serial_in_out.as_raw_fd() as u64); + serial.process(&spurious_ev, &mut event_manager); + + // Try to modify the input event. Will result in Error since the serial input was unregistered. + event_manager + .modify(serial_in_out.as_raw_fd(), spurious_ev) + .unwrap_err(); + } + #[test] fn test_event_handling_err_and_hup() { let mut event_manager = EventManager::new().unwrap(); @@ -453,18 +612,15 @@ mod tests { EventFd::new(libc::EFD_NONBLOCK).unwrap(), Box::new(serial_in_out.clone()), Box::new(serial_in_out.clone()), + Some(EventFd::new(libc::EFD_NONBLOCK).unwrap()), ); // Check that the interest list contains one event set. - let expected_medium_bytes = [b'a'; 32]; - assert_eq!(serial.interest_list().len(), 1); + let expected_medium_bytes = [b'a'; FIFO_SIZE]; + assert_eq!(serial.interest_list().len(), 2); { let mut guard = serial_in_out.internal.lock().unwrap(); - // Write 33 bytes to the serial console. `IN` handling consumes 32 bytes from the - // inflight bytes. Add one more byte to be able to process it in a following - // processing round. guard.read_buf.write_all(&expected_medium_bytes).unwrap(); - guard.read_buf.write_all(&[b'a']).unwrap(); } assert!(serial.in_buffer.is_empty()); @@ -472,11 +628,9 @@ mod tests { EventSet::ERROR | EventSet::HANG_UP, serial_in_out.as_raw_fd() as u64, ); - // Process 32 bytes. - serial.process(&err_hup_ev, &mut event_manager); - // Process one more byte left. + serial.process(&err_hup_ev, &mut event_manager); - assert_eq!(serial.in_buffer.len(), expected_medium_bytes.len() + 1); + assert_eq!(serial.in_buffer.len(), expected_medium_bytes.len()); serial.in_buffer.clear(); // Process one more round of `EventSet::HANG_UP`. @@ -541,40 +695,6 @@ mod tests { assert_eq!(data[0], 0); } - #[test] - fn test_serial_recv_bytes() { - // Exercise bytes retrieval without any input. - { - let mut serial = Serial::new(EventFd::new(libc::EFD_NONBLOCK).unwrap(), None, None); - - let count = serial.recv_bytes().unwrap(); - assert!(count == 0); - } - - // Prepare the input buffer and send bytes on the "medium". - { - let serial_in_out = SharedBuffer::new(); - let mut serial = Serial::new_in_out( - EventFd::new(libc::EFD_NONBLOCK).unwrap(), - Box::new(serial_in_out.clone()), - Box::new(serial_in_out.clone()), - ); - - // Serial `recv_bytes` consumes chunks of 32 bytes from the serial input. - // Write 33 bytes to assert on the consumption of only 32 bytes later on. - let expected_medium_bytes = [b'a'; 32]; - { - let mut guard = serial_in_out.internal.lock().unwrap(); - guard.read_buf.write_all(&expected_medium_bytes).unwrap(); - guard.read_buf.write_all(&[b'a']).unwrap(); - } - - let count = serial.recv_bytes().unwrap(); - assert!(serial.in_buffer.len() == count); - assert!(serial.in_buffer == expected_medium_bytes); - } - } - #[test] fn test_serial_input() { let intr_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap(); @@ -584,6 +704,7 @@ mod tests { intr_evt.try_clone().unwrap(), Box::new(serial_in_out.clone()), Box::new(serial_in_out.clone()), + Some(EventFd::new(libc::EFD_NONBLOCK).unwrap()), ); // Write 1 to the interrupt event fd, so that read doesn't block in case the event fd @@ -708,4 +829,78 @@ mod tests { // metric stays the same. assert_eq!(missed_writes_before, missed_writes_after - 1); } + + #[test] + fn test_raw_input_err() { + let mut serial = Serial::new_sink(EventFd::new(libc::EFD_NONBLOCK).unwrap()); + let input = [0u8; FIFO_SIZE + 1]; + serial.raw_input(&input).unwrap_err(); + } + + #[test] + fn test_serial_in_buffer_limit() { + let mut serial_in_out = SharedBuffer::new(); + serial_in_out.set_loopback(true); + + let mut serial = Serial::new_in_out( + EventFd::new(libc::EFD_NONBLOCK).unwrap(), + Box::new(serial_in_out.clone()), + Box::new(serial_in_out.clone()), + Some(EventFd::new(libc::EFD_NONBLOCK).unwrap()), + ); + + // Send more than buffer capacity bytes. + let stdin_bytes = vec![0u8; FIFO_SIZE + 1]; + serial_in_out.write_all(&stdin_bytes).unwrap(); + let mut count = serial.recv_bytes().unwrap(); + // Assert that the buffer is full, without the extra bytes + // written to the standard input. + assert_eq!(serial.in_buffer.len(), FIFO_SIZE); + assert_eq!(count, FIFO_SIZE); + serial.in_buffer.clear(); + + // Send an amount of bytes which does not fill up the buffer. + let chars_count = 10; + serial_in_out + .write_all(&stdin_bytes[..FIFO_SIZE - chars_count - 1]) + .unwrap(); + count = serial.recv_bytes().unwrap(); + assert_eq!(serial.in_buffer.len(), FIFO_SIZE - chars_count); + assert_eq!(count, FIFO_SIZE - chars_count); + + // Send the rest of the bytes which will fill up the buffer. + serial_in_out + .write_all(&stdin_bytes[FIFO_SIZE - chars_count..]) + .unwrap(); + count = serial.recv_bytes().unwrap(); + assert_eq!(serial.in_buffer.len(), FIFO_SIZE); + assert_eq!(count, chars_count); + + // Send and read more than the buffer size. + // Assert that the buffer stays at its maximum capacity. + serial_in_out.write_all(&stdin_bytes).unwrap(); + serial.recv_bytes().unwrap_err(); + assert_eq!(serial.in_buffer.len(), FIFO_SIZE); + + // Process part of the buffer, until its last byte. + for i in 0..FIFO_SIZE - 1 { + serial.handle_read(DATA); + assert_eq!(serial.avail_buffer_capacity(), i + 1); + } + + // Process the last byte and assert that the stdin was kicked for more bytes. + serial.handle_read(DATA); + assert_eq!(serial.buffer_ready_evt.as_ref().unwrap().read().unwrap(), 1); + } + + #[test] + #[should_panic] + fn test_avail_buffer_capacity_panic() { + let mut serial = Serial::new_sink(EventFd::new(libc::EFD_NONBLOCK).unwrap()); + let input = vec![0u8; FIFO_SIZE + 1]; + serial.in_buffer.extend(&input); + + // This should panic since it tries to + serial.avail_buffer_capacity(); + } } diff --git a/src/devices/tests/integration_tests.rs b/src/devices/tests/integration_tests.rs index b3e9aed48e9..8852f8b8df6 100644 --- a/src/devices/tests/integration_tests.rs +++ b/src/devices/tests/integration_tests.rs @@ -11,21 +11,22 @@ use devices::legacy::Serial; use devices::BusDevice; use polly::event_manager::EventManager; use serial_utils::MockSerialInput; -use utils::epoll::{EpollEvent, EventSet}; use utils::eventfd::EventFd; #[test] -fn test_issue_serial_hangup_anon_pipe() { +fn test_issue_serial_hangup_anon_pipe_while_registered_stdin() { let mut fds: [c_int; 2] = [0; 2]; let rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; assert!(rc == 0); // Serial input is the reading end of the pipe. let serial_in = MockSerialInput(fds[0]); + let kick_stdin_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap(); let serial = Arc::new(Mutex::new(Serial::new_in_out( EventFd::new(libc::EFD_NONBLOCK).unwrap(), Box::new(serial_in), Box::new(io::stdout()), + Some(kick_stdin_evt.try_clone().unwrap()), ))); // Make reading fd non blocking to read just what is inflight. @@ -33,10 +34,7 @@ fn test_issue_serial_hangup_anon_pipe() { let mut rc = unsafe { libc::fcntl(fds[0], libc::F_SETFL, flags | libc::O_NONBLOCK) }; assert!(rc == 0); - // Write some dummy data on the writing end of the pipe to handle it later on. - // 33 bytes are read in two rounds of serial input processing because - // it is handled in batches of 32 bytes at maximum. - const BYTES_COUNT: usize = 33; + const BYTES_COUNT: usize = 65; // Serial FIFO_SIZE + 1. let mut dummy_data = [1u8; BYTES_COUNT]; rc = unsafe { libc::write( @@ -49,45 +47,259 @@ fn test_issue_serial_hangup_anon_pipe() { // Register the reading end of the pipe to the event manager, to be processed later on. let mut event_manager = EventManager::new().unwrap(); - event_manager - .register( - fds[0], - EpollEvent::new(EventSet::IN, fds[0] as u64), - serial.clone(), - ) - .unwrap(); - - let mut ev_count = 1; - while ev_count != 0 { - // `EventSet::IN` was received. - ev_count = event_manager.run_with_timeout(0).unwrap(); + event_manager.add_subscriber(serial.clone()).unwrap(); + + // `EventSet::IN` was received on stdin. The event handling will consume + // 64 bytes from stdin. The stdin monitoring is still armed. + let mut ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); + + let mut data = [0u8; BYTES_COUNT]; + + // On the main thread, we will simulate guest "vCPU" thread serial reads. + let data_bus_offset = 0; + for i in 0..BYTES_COUNT - 1 { + serial + .lock() + .unwrap() + .read(data_bus_offset, &mut data[i..=i]); } + assert!(data[..31] == dummy_data[..31]); + assert!(data[32..64] == dummy_data[32..64]); + + // The avail capacity of the serial FIFO is 64. + // Read the 65th from the stdin through the kick stdin event triggered by 64th of the serial + // FIFO read, or by the armed level-triggered stdin monitoring. Either one of the events might + // be handled first. The handling of the second event will find the stdin without any pending + // bytes and will result in EWOULDBLOCK. Usually, EWOULDBLOCK will reregister the stdin, but + // since it was not unregistered before, it will do a noop. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 2); + + // The avail capacity of the serial FIFO is 63. + rc = unsafe { + libc::write( + fds[1], + dummy_data.as_mut_ptr() as *const c_void, + dummy_data.len(), + ) as i32 + }; + assert!(dummy_data.len() == rc as usize); + + // Writing to the other end of the pipe triggers handling a stdin event. + // Now, 63 bytes will be read from stdin, filling up the buffer. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); + + // Close the writing end (this sends an HANG_UP to the reading end). + // While the stdin is registered, this event is caught by the event manager. + rc = unsafe { libc::close(fds[1]) }; + assert!(rc == 0); + + // This cycle of epoll has two important events. First, the received HANGUP and second + // the fact that the FIFO is full, so even if the stdin reached EOF, there are still + // pending bytes to be read. We still unregister the stdin and keep reading from it until + // we get all pending bytes. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); + + // Free up 64 bytes from the serial FIFO. + for i in 0..BYTES_COUNT - 1 { + serial + .lock() + .unwrap() + .read(data_bus_offset, &mut data[i..=i]); + } + + // Process the kick stdin event generated by the reading of the 64th byte of the serial FIFO. + // This will consume some more bytes from the stdin while the stdin is unregistered. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); + + // Two more bytes left. At the 2nd byte, another kick read stdin event is generated, + // trying to fill again the serial FIFO with more bytes. + for i in 0..2 { + serial + .lock() + .unwrap() + .read(data_bus_offset, &mut data[i..=i]); + } + + // We try to read again, but we detect that stdin received previously EOF. + // This can be deduced by reading from a non-blocking fd and getting 0 bytes as a result, + // instead of EWOUDBLOCK. We unregister the stdin and the kick stdin read evt. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); + + // Nothing can interrupt us. + ev_count = event_manager.run_with_timeout(1).unwrap(); + assert_eq!(ev_count, 0); +} + +#[test] +fn test_issue_hangup() { + let mut fds: [c_int; 2] = [0; 2]; + let rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; + assert!(rc == 0); + + // Serial input is the reading end of the pipe. + let serial_in = MockSerialInput(fds[0]); + let kick_stdin_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap(); + let serial = Arc::new(Mutex::new(Serial::new_in_out( + EventFd::new(libc::EFD_NONBLOCK).unwrap(), + Box::new(serial_in), + Box::new(io::stdout()), + Some(kick_stdin_evt.try_clone().unwrap()), + ))); + + // Make reading fd non blocking to read just what is inflight. + let flags = unsafe { libc::fcntl(fds[0], libc::F_GETFL, 0) }; + let mut rc = unsafe { libc::fcntl(fds[0], libc::F_SETFL, flags | libc::O_NONBLOCK) }; + assert!(rc == 0); + + // Close the writing end (this sends an HANG_UP to the reading end). + // While the stdin is registered, this event is caught by the event manager. + rc = unsafe { libc::close(fds[1]) }; + assert!(rc == 0); + + // Register the reading end of the pipe to the event manager, to be processed later on. + let mut event_manager = EventManager::new().unwrap(); + event_manager.add_subscriber(serial.clone()).unwrap(); + + let mut ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); + + // Nothing can interrupt us. + ev_count = event_manager.run_with_timeout(1).unwrap(); + assert_eq!(ev_count, 0); +} + +#[test] +fn test_issue_serial_hangup_anon_pipe_while_unregistered_stdin() { + let mut fds: [c_int; 2] = [0; 2]; + let rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; + assert!(rc == 0); + + // Serial input is the reading end of the pipe. + let serial_in = MockSerialInput(fds[0]); + let kick_stdin_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap(); + let serial = Arc::new(Mutex::new(Serial::new_in_out( + EventFd::new(libc::EFD_NONBLOCK).unwrap(), + Box::new(serial_in), + Box::new(io::stdout()), + Some(kick_stdin_evt.try_clone().unwrap()), + ))); + + // Make reading fd non blocking to read just what is inflight. + let flags = unsafe { libc::fcntl(fds[0], libc::F_GETFL, 0) }; + let mut rc = unsafe { libc::fcntl(fds[0], libc::F_SETFL, flags | libc::O_NONBLOCK) }; + assert!(rc == 0); + + const BYTES_COUNT: usize = 65; // Serial FIFO_SIZE + 1. + let mut dummy_data = [1u8; BYTES_COUNT]; + rc = unsafe { + libc::write( + fds[1], + dummy_data.as_mut_ptr() as *const c_void, + dummy_data.len(), + ) as i32 + }; + assert!(dummy_data.len() == rc as usize); + + // Register the reading end of the pipe to the event manager, to be processed later on. + let mut event_manager = EventManager::new().unwrap(); + event_manager.add_subscriber(serial.clone()).unwrap(); + + // `EventSet::IN` was received on stdin. The event handling will consume + // 64 bytes from stdin. The stdin monitoring is still armed. + let mut ev_count = event_manager.run_with_timeout(0).unwrap(); + assert_eq!(ev_count, 1); + let mut data = [0u8; BYTES_COUNT]; // On the main thread, we will simulate guest "vCPU" thread serial reads. let data_bus_offset = 0; - for i in 0..BYTES_COUNT { + for i in 0..BYTES_COUNT - 1 { serial .lock() .unwrap() .read(data_bus_offset, &mut data[i..=i]); } - // We need to assert on a maximum of 32 bytes slices, because this is the maximum - // rust can compare. assert!(data[..31] == dummy_data[..31]); - assert!(data[32] == dummy_data[32]); + assert!(data[32..64] == dummy_data[32..64]); + + // The avail capacity of the serial FIFO is 64. + // Read the 65th from the stdin through the kick stdin event triggered by 64th of the serial + // FIFO read, or by the armed level-triggered stdin monitoring. Either one of the events might + // be handled first. The handling of the second event will find the stdin without any pending + // bytes and will result in EWOULDBLOCK. Usually, EWOULDBLOCK will reregister the stdin, but + // since it was not unregistered before, it will do a noop. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 2); + + // The avail capacity of the serial FIFO is 63. + rc = unsafe { + libc::write( + fds[1], + dummy_data.as_mut_ptr() as *const c_void, + dummy_data.len(), + ) as i32 + }; + assert!(dummy_data.len() == rc as usize); + + // Writing to the other end of the pipe triggers handling an stdin event. + // Now, 63 bytes will be read from stdin, filling up the buffer. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); + + // Serial FIFO is full, so silence the stdin. We do not need any other interruptions + // until the serial FIFO is freed. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); // Close the writing end (this sends an HANG_UP to the reading end). + // While the stdin is unregistered, this event is not caught by the event manager. rc = unsafe { libc::close(fds[1]) }; assert!(rc == 0); - // `EventSet::HANG_UP` was received. - let ev_count = event_manager.run().unwrap(); - assert!(ev_count == 1); + // This would be a blocking epoll_wait, since the buffer is full and stdin is unregistered. + // There is no event that can break the epoll wait loop. + ev_count = event_manager.run_with_timeout(0).unwrap(); + assert_eq!(ev_count, 0); + + // Free up 64 bytes from the serial FIFO. + for i in 0..BYTES_COUNT - 1 { + serial + .lock() + .unwrap() + .read(data_bus_offset, &mut data[i..=i]); + } + + // Process the kick stdin event generated by the reading of the 64th byte of the serial FIFO. + // This will consume some more bytes from the stdin. Keep in mind that the HANGUP event was + // lost and we do not know that the stdin reached EOF. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); + + // Two more bytes left. At the 2nd byte, another kick read stdin event is generated, + // trying to fill again the serial FIFO with more bytes. Keep in mind that the HANGUP event was + // lost and we do not know that the stdin reached EOF. + for i in 0..2 { + serial + .lock() + .unwrap() + .read(data_bus_offset, &mut data[i..=i]); + } + + // We try to read again, but we detect that stdin received previously EOF. + // This can be deduced by reading from a non-blocking fd and getting 0 bytes as a result, + // instead of EWOUDBLOCK. We unregister the stdin and the kick stdin read evt. + ev_count = event_manager.run().unwrap(); + assert_eq!(ev_count, 1); - // Serial input was unregistered. - let ev_count = event_manager.run_with_timeout(1).unwrap(); - assert!(ev_count == 0); + // Nothing can interrupt us. + ev_count = event_manager.run_with_timeout(0).unwrap(); + assert_eq!(ev_count, 0); } diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs index 28c55a74ab1..61bf53981e3 100644 --- a/src/vmm/src/builder.rs +++ b/src/vmm/src/builder.rs @@ -191,6 +191,12 @@ impl VmmEventsObserver for SerialStdin { self.0.lock().set_raw_mode().map_err(|e| { warn!("Cannot set raw mode for the terminal. {:?}", e); e + })?; + + // Set non blocking stdin. + self.0.lock().set_non_block(true).map_err(|e| { + warn!("Cannot set non block for the terminal. {:?}", e); + e }) } fn on_vmm_stop(&mut self) -> std::result::Result<(), utils::errno::Error> { @@ -238,7 +244,7 @@ fn create_vmm_and_vcpus( Box::new(SerialStdin::get()), Box::new(io::stdout()), ) - .map_err(StartMicrovmError::Internal)?; + .map_err(Internal)?; // x86_64 uses the i8042 reset event as the Vmm exit event. let reset_evt = exit_evt .try_clone() @@ -556,7 +562,13 @@ pub fn setup_serial_device( out: Box, ) -> super::Result>> { let interrupt_evt = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; - let serial = Arc::new(Mutex::new(Serial::new_in_out(interrupt_evt, input, out))); + let kick_stdin_read_evt = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; + let serial = Arc::new(Mutex::new(Serial::new_in_out( + interrupt_evt, + input, + out, + Some(kick_stdin_read_evt), + ))); if let Err(e) = event_manager.add_subscriber(serial.clone()) { // TODO: We just log this message, and immediately return Ok, instead of returning the // actual error because this operation always fails with EPERM when adding a fd which diff --git a/tests/integration_tests/build/test_coverage.py b/tests/integration_tests/build/test_coverage.py index 02dceec875e..0d87a1f0fde 100644 --- a/tests/integration_tests/build/test_coverage.py +++ b/tests/integration_tests/build/test_coverage.py @@ -23,7 +23,7 @@ # this contains the frequency while on AMD it does not. # Checkout the cpuid crate. In the future other # differences may appear. -COVERAGE_DICT = {"Intel": 85.10, "AMD": 84.26, "ARM": 82.58} +COVERAGE_DICT = {"Intel": 84.9, "AMD": 84.08, "ARM": 82.58} PROC_MODEL = proc.proc_type() COVERAGE_MAX_DELTA = 0.05