From 832032c2b5f3c9c81084e6776134c883fec94f5d Mon Sep 17 00:00:00 2001 From: WeiXin Date: Tue, 19 Jan 2021 21:35:52 +0800 Subject: [PATCH] =?UTF-8?q?[cherry=20pick]=E4=BF=AE=E5=A4=8Dsave/load?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E7=9A=84=E4=B8=A4=E4=B8=AAbug=20(#30543)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原始PR：#30485，#30507 --- python/paddle/fluid/io.py | 19 +++++- .../tests/unittests/test_paddle_save_load.py | 7 ++- .../tests/unittests/test_static_save_load.py | 55 +++++++++++++++--- python/paddle/framework/io.py | 14 ++++- .../static_mode_white_list.cpython-35.pyc | Bin 0 -> 19792 bytes tools/static_mode_white_list.pyc | Bin 0 -> 21803 bytes 6 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 tools/__pycache__/static_mode_white_list.cpython-35.pyc create mode 100644 tools/static_mode_white_list.pyc diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index d5963675a8..313855b6c5 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -22,6 +22,7 @@ import logging import pickle import contextlib from functools import reduce +import sys import numpy as np import math @@ -1715,7 +1716,7 @@ def _unpack_saved_dict(saved_obj): unpack_infor = {} for key, value in saved_obj.items(): if isinstance(value, np.ndarray): - MAX_NUMBER_OF_ELEMENT = 2**22 + MAX_NUMBER_OF_ELEMENT = int((2**30 - 1) / value.dtype.itemsize) num_element = np.prod(value.shape) if num_element > MAX_NUMBER_OF_ELEMENT: unpack_infor[key] = {} @@ -1809,8 +1810,18 @@ def save(program, model_path): parameter_list = list(filter(is_parameter, program.list_vars())) param_dict = {p.name: get_tensor(p) for p in parameter_list} param_dict = _unpack_saved_dict(param_dict) - with open(model_path + ".pdparams", 'wb') as f: - pickle.dump(param_dict, f, protocol=2) + + # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3.5/6' + if sys.platform == 'darwin' and sys.version_info.major == 3 and ( + sys.version_info.minor == 5 or sys.version_info.minor == 6): + pickle_bytes = pickle.dumps(param_dict, protocol=2) + with open(model_path + ".pdparams", 'wb') as f: + max_bytes = 2**30 + for i in range(0, len(pickle_bytes), max_bytes): + f.write(pickle_bytes[i:i + max_bytes]) + else: + with open(model_path + ".pdparams", 'wb') as f: + pickle.dump(param_dict, f, protocol=2) optimizer_var_list = list( filter(is_belong_to_optimizer, program.list_vars())) @@ -2169,6 +2180,7 @@ def load_program_state(model_path, var_list=None): with open(parameter_file_name, 'rb') as f: para_dict = pickle.load(f) if six.PY2 else pickle.load( f, encoding='latin1') + para_dict = _pack_loaded_dict(para_dict) opt_file_name = model_prefix + ".pdopt" if os.path.exists(opt_file_name): @@ -2220,6 +2232,7 @@ def set_program_state(program, state_dict): static.set_program_state(prog, program_state) """ + state_dict = _pack_loaded_dict(state_dict) parameter_list = list(filter(is_persistable, program.list_vars())) used_para_list = {} diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py index 3d5c8dfb48..3a8531db6f 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py @@ -16,6 +16,7 @@ from __future__ import print_function import unittest import numpy as np +import os import paddle import paddle.nn as nn import paddle.optimizer as opt @@ -90,13 +91,13 @@ class TestSaveLoadLargeParameters(unittest.TestCase): layer = LayerWithLargeParameters() save_dict = layer.state_dict() - path = "test_paddle_save_load_large_param_save/layer" + ".pdparams" + path = os.path.join("test_paddle_save_load_large_param_save", + "layer.pdparams") paddle.save(layer.state_dict(), path) dict_load = paddle.load(path) # compare results before and after saving for key, value in save_dict.items(): - self.assertTrue( - np.sum(np.abs(dict_load[key] - value.numpy())) < 1e-15) + self.assertTrue(np.array_equal(dict_load[key], value.numpy())) class TestSaveLoad(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load.py b/python/paddle/fluid/tests/unittests/test_static_save_load.py index 0f4fca6d7f..68d0e07e0c 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load.py @@ -1324,7 +1324,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): name="static_save_load_large_x", shape=[None, 10], dtype='float32') - z = paddle.static.nn.fc(x, LARGE_PARAM) + z = paddle.static.nn.fc(x, LARGE_PARAM, bias_attr=False) place = paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) @@ -1334,16 +1334,55 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): result_z = exe.run(program=prog, feed={"static_save_load_large_x": inputs}, fetch_list=[z.name]) - path = "test_static_save_load_large_param/static_save" + base_map = {} + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimizer var have been update + self.assertTrue(np.sum(np.abs(t)) != 0) + base_map[var.name] = t + + path = os.path.join("test_static_save_load_large_param", + "static_save") paddle.fluid.save(prog, path) + # set var to zero + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + ten.set(np.zeros_like(np.array(ten)), place) + + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.fluid.load(prog, path) - result_load = exe.run(program=prog, - feed={"static_save_load_large_x": inputs}, - fetch_list=[z.name]) - # compare results before and after saving - self.assertTrue( - np.sum(np.abs(result_z[0] - result_load[0])) < 1e-15) + + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + base_t = base_map[var.name] + self.assertTrue(np.array_equal(new_t, base_t)) + + # set var to zero + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + ten.set(np.zeros_like(np.array(ten)), place) + + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + self.assertTrue(np.sum(np.abs(new_t)) == 0) + + program_state = fluid.load_program_state(path) + fluid.set_program_state(prog, program_state) + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + base_t = base_map[var.name] + self.assertTrue(np.array_equal(new_t, base_t)) class TestProgramStateOldSaveSingleModel(unittest.TestCase): diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index 66f843dc05..2dfad8dc10 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -19,6 +19,7 @@ import collections import pickle import six import warnings +import sys import paddle @@ -262,8 +263,17 @@ def save(obj, path): saved_obj = _build_saved_state_dict(obj) saved_obj = _unpack_saved_dict(saved_obj) - with open(path, 'wb') as f: - pickle.dump(saved_obj, f, protocol=2) + # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3.5/6' + if sys.platform == 'darwin' and sys.version_info.major == 3 and ( + sys.version_info.minor == 5 or sys.version_info.minor == 6): + pickle_bytes = pickle.dumps(saved_obj, protocol=2) + with open(path, 'wb') as f: + max_bytes = 2**30 + for i in range(0, len(pickle_bytes), max_bytes): + f.write(pickle_bytes[i:i + max_bytes]) + else: + with open(path, 'wb') as f: + pickle.dump(saved_obj, f, protocol=2) def load(path, **configs): diff --git a/tools/__pycache__/static_mode_white_list.cpython-35.pyc b/tools/__pycache__/static_mode_white_list.cpython-35.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7dae6374903af069c516bb436c2fcea7f761dcd1 GIT binary patch literal 19792 zcmeHPb(}m$m48)O2n2$KV8I~)lJNWeLU1QQAOuOUMVj9E_RKarGd<~^+5K&BcXvC^ zIqu-x{ovdMxyvbb?tYwb-&gOo%N_;K7222p%eUnBd`p zM+hD%c$DBG!J`F_5j<9KvEXrn#|thIJV9`&;E94K37#x?ir_NA<$|XQo+fy@;2DBv z3bq8#5#M%EqIOKwSw0PUN3lq;EjSe z3EnJti{Pz-w+Y@Zc!%Jff_DktEqIUMy@K}%-Y@uo;Ddq>3BE?~VZql5zE1GZwh`(@F~Hk1)mXoR`A<`-w|9T_+7#834UMj2ZBEo{E^^ug3k-SAo!x-OM*WZ z{E6UC1%D>^bHQH-{!;K)g1;90jo@zue<%2R!9NK8QSeWKe-`|U;9mv*Cit@8-v$36 z_)o!q3BE%44+sG<;93FK4!BOhbpx&!a7w`S15ORNLBI_IP7AnEz>Nb=54cIdO#^Ng zaPxp$1e_6Y%YZWjZWVCrfZGJzHsE#vX9e6o;0^(I47gLkodfO?aMysl1>8O0?0|Cu z?h$a$fO`d;8*pC0`2qJ1I2Ld`;6%U$0rv^GFyOuc_Y1gxzyksv81SHg2M0VP;GqEz z3wU_IBLW^7@Th=`0v;Xkn1IIyTpaMYfX4@167Yn8O9P%5@T7nz2RtRZ0-hU?1w1bx57-VE1t6dZ7za!Ob^^+P=LhTtOao>C^MEQ~ z5pYF79nb_U16Bch0s8?b0}cYN40u7n3jVVe- zyf)x<0k02uL%q7g4fwHu9}oD6fS(Nbsen%ed@|st1AZppX9IpN z;O7H=A>bDSektIW1AZmoR|9@6;MW6wBj7g!ek8Q*KMMF{6niFj+o+alf`@s5ahM!YNH-4XAJcyGk}BHkbIfrt-Ad?-#n61Rj7B3^d(hZziP zov|#MWtNY}<-Ev7JNbNGOtWgSb)y6yRr684)bTCTxR@5RV!qrjnt0tS4yT$Q-X38(-wSKdlXQO(oGdD?VrWLZfXo~dVn>(o5T`jU@zCA4z z@w5b8>c-Cc6>%6UMiYr#de8h{Tdu-j*&<4=GP1vHUrheIOb#Zc$&ynTL z=C(Gm3FWEu2u9g%lhxeeMwbpgIP}JfW?J&--DSI>qb+jIE~^a1V!5Mex3Sw@vC8WL z+4OjZvRUSOeNLX9CiBr~Rp+Avg&0Tk(X!mjmt{52=JRY)=V(!pqnIvpMK$u~$aP); zH?-Z5&vbOV;b>XTO7c4!FHRiiKQ6qWUp>oDX8EFY+2VEz;Cf@Gj`wwF1mbYPqfyTG zCYf^O7&d2gxzDsm00B*zu0zIGDCZO1^45Mrl`CGWR-zSi5apozI%erIH#1aIh)G?h z`#Nws9Zf5W{xm-*>a9B&=4?S3E=Icuh_ z$9ws7l~%d#Q)lLhV$)`sNZ?0XjdZDt2?q^{$y+T{x;SxryTD#W644@1%=G?cJ}s|I z^5lfw&PTiZc@4kgcHWeutfs8Z^I75Ax1BFXJK4Od)8}uIHeQ2ed&k@D_#N_~ah7fu z6v2jegzHg|;pt*V&fy%-Yq#AlM=So9XQicB=sj*_pIe9TB|~L*-!7*)k@IQRsi4^@ zPM8f-p*9igK)%22W2EZlNN=)~gsiX{4Xo>Sxyb9KvUHXR!owLWwoNomhqtSf*=Sm( zv->#|!mB5_I?Jv@kkO+~0_m*hj=ssFPGnR9m{p|2j*|YDl!mLWpLQALP2sxEKhsHV z`G`3DTw;VncB*O7>?X_N8}6*;yIDbL(~p>rj}nYS6UcOp?Sqc(tEnNE+Y>9in~ z#*4X9jOv|F8{N_PIjUwPT0wmvEfe!id=y?zDXCmTt2Nv58&z|3E9Xd-f(P85R-@ha zP!;cXa!DS{^9j}5XiN{XcZ^mwX}(!n2Q8_>4Ug5Dg&v>2$?#1b72n)dznrDXkV{4NtT1lw;^K}-#-*2#6>kgco+%Ic(%5jncGfg8p^`=ZZ>&&2a zrSwjw)i#N`m)9*x>D`kwx;b_}lbd=Q)7FuWTr+y^GHz%x3L7U61}aJ99&TdjY+sw; zJ}!3&%%O8)Im&3KCi8Z@nl-C|=^O3oz?A^`GIxsWpVKNi5SpB+T46>huKI(?-J;;l zYHH-xZIdDm=BO0?tep4r(ekI{*k%ZbD0GHg^_CGw6={L^+#PTWz7vdY2rVV>q}>zHhNwZrwG}qfi@1<(w8N z1fXDS7)A448)yl?Pz`TfH8v0?^(vcFHB9_BCR3cDvq$dDFgb;7dUNC1#&U|=s;+r) zCF_wN*x#Y)*I6?P1+TOg44a+}lNpqIgWU#M&Uji;k*{vHBR0QvcvFw#j+kJOck~Ko zK1WZik{2~@4iDa1>BApcR4JN1#n?Q{TWYH^Bmy&TTbCoV^u=sJP1j^ik78o07Wg1z zc2#@)&;yRU9u$_rImJ(sM;hpAFig>QGgS=1D|$?K&e@0>U3hrkA$e3?PXrUlYcpS7w1y_RgdF)z)))rJM;Q(KH-n zBv^$EV|bptBU44g$LIQW)bZrb*gprOl5S&^5L|oE~YD9pM{B3 zD$)}h&K*tWxf+?8F{>o?H3;6<{Gx6{l-V}3yDL&OqO@=|Y3JOKQH9#n)rDl$XhB1r z5Eof|$qg0OB6d3|ux2$NMKIhc9i6MX+b9ChknW%b5thAL95YM4GjX&S;k73HaCceX z!lo&>+{?>q=GxmGymZ6tFY_Xroqduocc@>>x)>OmNxot}pY!xIvQ>&+%^7-`L9@ta zml?=*VF{DzUpLz(Qv(bm{UMB$Zdq{DmoWN})-`iNyJG zhKhBypU{k7og3BE&xJ3QUYdNycsFZ>+|_z8l~Uez69&g!DrshIQV(DmiwI`0T`l~7DPw3?+%@Y)f^ON>bk=iF{PCQwAHp}#0oRMm9yhrosU41xpKykixI7+ z*1l4;#PxUp1~Z7!*a>>sYnzS@u+e8B`dD&p)yty^Xc;7 z@a8Eq*PM%jv3V2o&gFbm+sP~hDOr;|nYq+cV%DHZcT~7sj^<tEoSR6p zmX^Eom2B_$=p)-jcg-5z6JdKAe?~X%CYfHFH@WMoI3>tV7B2LkfsV_ZH%XUP8=5>`pa7&|;W!$85-M zO}D|3U155xyRES_S>>#mRh8B5ZP7HRzw5S5wWDsG+1~+d#?LbPC%QS2!PNHn*o9d? zZ)RmM&*YKRwd+q=$(Uz->xPOD{brqRxXlOY)s z9??8Y8fnt#k!_0byE#@Mm3KKaU<$`<;2%^e^MzLI^}2opg{4yXw3ttfiYCmdeU!8I zn7T!7kb1rLRn?+#hHAN?WuCEEAuiY+Miasc5p#i-YSo}0<rfSjD5Fim?=z| zmJrLi?d5hS97icUwCm0CdcrJ`zMU@ICCj9Z(Sdp=##77%{CI>h%hIjQKl#``-7Txj zlPz8Co3nz(tgKcZ?0EgAE;P6+PL8H4meW&CqWOdGY&9OUwk?SHMphy*%m_Funw_?H zVytIhqhM%WGp0M4oO;6cW0Z4d0UcdtrMX|TLbP3UleBO7LuW>kQ+ORQreV>L{PO z8r#5IKx|N{+SYC-*#V`oZKc^iMy*PMYqVmSFZ1J>*tURSN=}${lUi?* z&LEpv*{Dq%Nq+0SX+hD|rVmZbgmz;xBqgX!9Wz#d_T9^92bbHDv`msUTl08wt>deg zOleKqe)SMbV=1;W$fLGS0WEo(VA%JMLSdzMHTT%!@O;uv-mKA9ulB}OQ;(;>a;mj_ zlgLGRl1yxN7;sGsm-2zKmZZU z?4?A~tBWoZY(gI8^&zTP=d|5H3Wf7qnwM7R4 zPcnOA{i^Sbbm%r?wRUGz-8M4xK2~i@hiDnuH9B;6sRGPzE!s{CuQb~(=P6zD zh^wE|=we#zCApwQcdOgaFSM(lJf}wpgQ*q0-QtMvc@=Z2`ezY;zsNI$E*tl}r$=D1 zgCSjOX*`b@R>*pi@QKW-@at9>cNOm`i<21zCcVM4TUCewEa| zvp$OA{PA|4LQH4r){@V4>akHktGU;q+fdQ{L~Ok?_0?jI7A@>ai3>O>o`$=-avWz1}&A z@?_3T-{Y($;+C8DR5rubqK+&2gDxm@@;HypcMS4sA`c_=MmxDYmEX*=S?0PaJoTHc z0TtN-=c}(9D30G=g<9X;wDG1dT`*6y&XRUI2I{LzO;ehL<2i}*A%TeDJX$0*^i!tF zYgW)>@I_)pi|d5;iYy<~wP*0n*dy2-oYcx?eO1E0F?>|eP?(vpnNHe(eJ?(+vCAG( zFuTYSEElt#nZ(uTtRPdJWE$?0IKM2kg~kHVl1Uq7xn*w~ZQDJUtYuW`B~$y7!&GkA zP0}iujon(X?~%A{H`mT{y15VeoG{Ir(N5M`15xKgGDqogOvjp3#a4UMbBq1 z)XIcTL7TM)KkMdM&$iI`@G z_lt6}qs=ePx7t^BCQtk1y*|KW`Ow!IggYC;K{c(~hZ`nF3fVi+X}(#r{-thzD1&h) zJB7Z}s0ApxOqOyHAd5oJhy|d6>1xO5?~(W5d2X6)Rbm`XNiin9X7#- zR+|YNps`LsA3x{2FHfrnbJd7_?6wq;CC79a$EFdsQn{FR&r{#_E=MzDwSPCmA7Bg#W_ z^r9Bso2@+}htp$OQQIrzUlk|I`3`_dUA4d5@w=u5cWX|KkJ|0aSbOGcWa|@}J3G8H zAGjKuv^Bym%3`&3@AVwAjd|IYXi-xM#luNmasU6>igc(!`%{;iDM+Q%=4$hL>$z)= z?JVe;K*gNpj#OyvRG^=$^+trd*KJ>^kIR}BqGnli-oKq;3&|(iP{g0RYvZr=p)In9 zu!q4ZMBV&o>z;WWOE9}*(*EyuZ?ZLEGs~q%NgZkJ=Cl*q0KJe z()Q~Ht?cBaHhvn>Z(un-2p4MejvMNU$#L}&XaG?LdI7_evGnHD=E=QUlUSo;H?cKh@k$Ap7b zbVl9$rxwmVMC$ajE4f<`pao-hwd%wZEBHvmT;|{{d*78YALO5zZJ zUw6cI`o?%;Nw=NOZG6h8sORNezg;ig1ozD75&CtkO!Z*9@ZkMZKV^*B5c$DU6XMPJ zjjJ5qM2q_UM-FeI^BbFVn$WN7M1BL+%-Kf#(l4{)Y{osC^DbtZgc;kYW=qQmkj%IEsH8_%a=(pRPNxGIyo3#}CHB_&G!rC#s z+PU$ywYiNW+O2$De+)Z0?TEc7?k9|>b^qA7pE5SL-mQn5!ng2^@4I(pGS@Hg{E)B> zWD}TPV>a%&IlZyEO@7*+6!ago$kxZnybON2)pOw7&j{?c=|#q(qS12U(y-ZX zF(9uAqB77spI(C^V6!x7;hBkp8DCw-*NrBxiCh1}WN-3+Pu+Cs|1M9|6>j61WqSd( zONQ(cLEuzC=HuC4Z8^(9PO?cTz%Oge{Bu7=SkJ$BJY;atV@rxN;LFMqb` z=(c}nIuo852#2LQUIN?IT;~(Sa=2Ol+#^AaL7v$U(T!$4VGZ5EHtw#Dw-+&SwxWGP zGyl-`fQBIdNp=F6|2pEeY`Vy$=Y*@yyzKIaUw-i;vnM?9Q5R*GUv$~!7hn3A?2?Nw zyZob4Uz`6oJn0`_e#H4TpMjmPEz{=wCy~I@;@ngU+Ab$PK7YxgVspONkF*4pv3jIs vME1STTO3?<%7X~xyRC=4X9}(-xW3>l!3_jA6x>K~W5G=XHx=AWaC5;e z1h*92N^onzZ3MR!+)i+N!5sv56x>O0XTeq3-$#k1*ZfD zf-40t7Q968Qo+jvFBiN*@Jhj}1g{pnM(|p}>jbYCyg~3r!J7nc7Q999R>9i@Zx_5n z@J_+I1n(BSNAO<3`vmV7d_eF)!G{DN7JNkTQNdRTJ|_4|!B+{sTJSZ3uN8cq;Ohk+ z7kq=@8wKAa_-4Vk2tFbBq~KG6PYXUH_^jYt1>Yw4oZ$01wSJAQNfQ1eq8Vqf}a%pl;EcYKO^{A!OsbPUhoToUljb3 z;FkrzBKTFouL*u#@Ed~P6#SOpw*|i=_+7#834UMj2ZBEo{E^^~1%D#=Q^B7J{#@`E zg1;2}mEf-hew&>UxNP@{Ey&E z2|&PzJmxU@u@Eun1TNQ~|4iD+20( zCZG*i2kZx&3^)~V5O8I{ivwN~@X~;n1-v}q6#=gdcvZlw16~vG+JM&uyguL!0dEX= zQ^1=8-V*TEfVTy_J>VSy?+kcXz`Fz96Y$=E_XWH^-~$044ERvMhXXzm@X>&;2>4jQ zR|b4lz*h%+O~BU%d|klT2Yfu>8v?#D;F|)zIpA9YJ`wQAfKLT{I^Z(_pAGodfNu-< zT)^i8zCGYO0=_fgy8^yD;ClkTH{c5a-xu)x0Y4D%g8@Gj@WTN=67Zt|KNj%g0Y4G& zlL0>!@Y4Z56Y#SEKNs-x0lyINivhnB@XGS@W%mv67Z)1e-`lP0e=zjmjQnj@Yex<6Y#eIe;4rg0sj#2j{#o{ z_@{t>4)~XVe+~GzfPWA8kAVLS_^*Kf4)~vdF9k?Mh=>tqL|h}{ni1EExOT*KBCZ>I z{eNb}^`h$kvuFz<&f?#r+JfuTB1GIM_7>sB5jWvanuds*Mch2%77@3MxK+fhBW@FM z+lbpm+&nh__l!6@;$9K=j<`?6eIw3^I5*_O5f6)ac*G+j9vShdh(|{}CgP%q$3{FZ z;_(p|M?4|oi4m7XJSpPRh$lxpCE}?OPm8!L;_`^6M?53qnGw&5cy`2dBAy%Zyol#V zydWZrcws~yu@fWJ4wyf)%>5wDMUL&O^+-W2iXh_^(%HR5d% zZ;yCK#5*J274hzf_e8um;(ZbCkN7~u2O~Zd@!^P%M0_;PJ|3SVG!XH!vp+^>puGjX zfwpMcET2xxWsyyG^X0ObXVt2`!GM}n%SqlU=#~Shm=}v;*`6$$BFll(;+pl2;%+s} zo0ju=c4bjlO*Sw0isKu|IS-K6dW3SBP3oy4Z#>MLSIFw3DbhJ^=FnPu|q3k7Wwe_+`8Tkw~y3FBu&AjB5#?&x~4_@UQ(^eUZRlBQ$Z)^ZFQd?wEiXZ?BTsZf;CEb3cB?v@70aT|TmFrQ zyq>L5=gV2q`d;-cTTt+N()mNxv!<$(*!jYBogbttW+!Xip9`D1$eV^|-3FDjrGiZK zI7&hL`Fx$G8)vGNE)#9%O`FK%M_x~K4QD$3Zb*Dyd!-yp+#T(D`xS{v4Mpx%8kPCH zyfQJUKg3Qx**lrna3bvFO*zSGvhXrr6fXEXc{|z7mQ|e&bn{`84RE%9e6%#T<;ykB z6gvf(yP<;PLMX^J>DosAF`D-GM`b7?PX{p_m9ykBu!+-qKCFrCJoOzXsV4_G{oI3QxQ6=GtV0f z5p;oy)TguM2((@_>rHh%*-yVs?2xy)zmxu&SIJ0HDbCfL zP%2#;jqrO#!K>HQ3hnI%86Kf2gFe7cjbdj#Qw}zbNV1Z7>7KdkVHQnEzFsBK=?ohu z-qrRxjLdZZ&IXT;Tuf;pk}s-UmhB(Y2Q$%zqgPE`mC3WpnfZ|(H!K#Af z{CoQ`Xyxd|@(PP$QPoKoCD*944pX{xBZC9_usJR1m2?kwn!&Cc5023DD)}$QvM8Df zRdcm0TzjKv^U3b`_dZRls|OVNJDbsWgu7=b^I29bb_%q2vTM+(tjnD>1>lg9Gux(~ zXE%tB4Q074W(Tg<4NHHs8J(%xROF~mE?3pMZf0<9#)?0cu6|urv+Fd~Y#|P)KUcfi zVm)umjL(|<1QSLaEY(YBO{)-)FM8LZX(^cM+$O6tT5_q5#enFU$2ssIq3Lk9$=t8j zZF~2DW`*iv>V?k~)&MT^ps~BJp z!hMfiz4wTZ#?fOw?L3C8r^T(4%V4iRzRh4+j@)onr8wp~#u;?9{9RPi(o@j)-DE>E zQ^jIM4WrSw9?pb7?ZrVx=eu^F%3;S{$cip7_ZK?Fm(}1xV-OV=Q=7#Yv!-v*dnMU~ znxyb(%f|H~O+js8O4E>2g6Tgkdz07R$0#stQy1;J9`GDWR@*OX4Nz3cnsv7zUGHyp z*P)%;YqqXOaP0O9-9Ynii-SZ_d$nm!5|6YjS1_U_oT+DG&KzztO(BlGXu7rA8u}Oc zigEkFs&H|h78;yrf~5Bbat+ViQRwjN<)Wx(B(2@7gRA%9=hFO4zI6WB#8B^OLLkYo z>B5xL;T8Pin@Juev++3xt8YzLxYVVR-7z0~%7}$mNIr|{kshcinzo#+t96s5$aRBz zhd~>ZoEOwt#`jbGNv=?@w@jJY&6{24iEc_w$A7`MtlfX!LS!el*IP5< zWS*pGG;}r@>!z(+(qMFFJ(j{6_tvM33Pv>J*@}}@y+>}MPowtG#c|psuh3tkt&?Uk zW}UA2q=UHoFolv@<_=8POQvwDCH+zLU5<|o_92ve!Ka;n;`H;!&bPZhQ&kW-57mb? zE5>*u5-66(?Gr%>Y1xo8wJzps(uc{cl$6vvWNFx)O|rNKGF7x)+R`&tT-dC%F2+%d z9meWcBo9sZb$J}ny`H7G4rvPKv~{C-Fg2^3YW_M`R`4)*{10X^(5(f`;_YzA1Ir|HQkyr zY_2v&7%+&NOr4}7)O65*F$2gXuY~9myB2@ zsYG++-8OEcRRDJ$+H;RMA&JE_GKKnN$zs89z>KNweBK@$xkbvqXgEw>+P;{1AmwsW zcc(NztYo(QRMtxpX}XO_f~kz@47i+fk7dOehSb|xPxWUrui9qx5;`VTdKOGB&a<*| z3vAh8T6kxpWzrIioRfcVNuY9o1IiUcWa`h_TFijG%hL0 z!D1jioCk1axygWQA*;!LbC6iAn#^583fPJNcKvPS5bB)V1&#Dh<>T9p2sUyH{H$OI9 z+SwslkE~f#Rad^U+4G#Xd%R1^J=y|hCr01}55k~}lgt8+ushRZ7i2!fo@z9N@jhR} zNHk|cYpMA=&0DZ8v&&5FoZRJ8yX?E?^zD`Vy!=K6-cnjbHd?oHQYBT!bDMEY0FCzh zNFjw(V>DCZ`tCXN8Dyo!rg2j>kgv32rNr<E~9f#`!^0PQ$|+lPItk9FI~6WMYv~*oeN>sOsBCSbAWKKp66SS$HcrN(+J*Yd+cz#`;8lJ*V zu_&l+OQxmF2kdixb)mj+acVMOvxJo5Lk;14mo7EfnTHXSv0qjqy|)p3Q8c?l(a~6l zMZkiAP)#rRRB~w296wmOWC%8<;G*o@4O5cTv2+EfjCVI&3E#HT=7BC-Z11qB=*G|j z`bNBFs==7^(-YKUmXw@Jp2t8we=^P6?Xm`|DGhE$r({GTMbcAC=~bt1g4UHrZFH@> z%=D=VLkgX2%18@?l-#<<S(0HHl27(7gCVk4Pyo%o z@k{r+FRrp4tO}%dCC?@Xnv6kge0$1t5)UIR)6H3baubdLww041xN9JBF-{pIqf7vwt}BOYp4%mslY0lwtbl>|m?;6lTtLQz;z_rs+M>ZN_#jM+XqfH3 z`gUW75+a43LtBM=r|O2X?#Cw{l3CE|PRC|;oTEmuTCrZb2X=T!(h$6HqrDv3CZpQo z_~7In9#7w~ca^%C9L?H!s&}{sK0Ix)|FvBt6I~QKF!{*Kq z)m985m`iFVqYM<`rx@xoJL=WL@uUvfRhImn>!Hl2x3wH^I@X9AtQe@H{LK!M?pK&| z$WWZ*g%>;JGDX|&wXO`U-q*a?Ph3WG$mJUb;3#oaF1WOSP5%!jRVDc7S z6-q(N@KjSebVao!Y3x~(yZVsRVT*=xlALF!4LyoUL7kw=#|c4b@6hfRbH0Th%;&H; zp85+EYVm23CGM#Kv#KFekvhuXwlAzJ(DVlFc5*hH)GVV> z^qwPUFPgMLm2lQQ2J@sAxulm!E>k(xmYnpPPJ8c7nwQec3WJ=oO
    uoSy@*$VX) zwLq!AxmE$gPL}RO& zi;g2U=uCW4^}ZV^x1+h}R9Q41vM<5=Wd`0f+LALFfppm*9?MjBvs;%GOs%B%%FAil z*;DF!t%q@v2}W;ewY7NYZob#>SZqUFXsh5xlBUbs_^X%#9!5_q+Bx%Tor-;%qglpL zs)bXw0;fiMGk8wFbeGOxz9WSJe(4RkPO+%v^ob@N=Co0v(PqHGo%aP+ObB$vTdfa{ zl!q>0BH5&2eE{k2mY^KxFS|B%(e3s(Y%1ZXF5Gx;cHhjYVap`W#?A`e&RzL6wvk=bX8Nh6UhLYJjwlk@ zxz|hjHHr2k(6s8F!>GxchbEUtRVHHxqUbC*n{~hYOl(Ef+<45#7JF4rB!}6swZ?KC z@7c}PE7p?w0*Y!`^<6l}s87YL0Z6q0DHHL{l$THTmK6v%cRxp~0E< z3|qO5F>0eA(P;aS*~deuLlpuO)6vi_!nBgzHs;V8rn5&QnNp(Dvm}u{$1*}!6SlDo z`JyZt-y_F$=@*T>*;$`;2eY=9(R!QXQYh=v$WV#>qMD&6+3Ybp8P%wV;KnIUKVm^n zruM`tTO~)}VV(tt_$*Nyp@VgKoa6(JBAd zd+0jpL0|Tybh=Teg3dPOlZpb3`>cnL&~@EtZpFhoE36hvro~d#U@S_GuJw*YnL;+y zv}A`Ltq>hdH?YH3Of1g!I8zQbQETyT77g6T21jC&bvg8_s4_+6xH#~-qyUHQxsv8+#cn4P|R@MBU*Czaf-+2vKw8D5OmCjbToycW20JwzVa9@pW{;mW`0O;%q$!!&tTvu$?kz z=WbV?@-e1-ZH>}; z38H;nNDiya5Yo;?*50f_Xb&o(MK~B2W>ia&EvO7`>cB(!DzqRu=dh5`3+kUp6%C1A@ z)qT+6IHP^-pL;mgm}4-_jxzl+ptg=R*wAj|BPm9sqxKigrPK$h2(PM7jDw(9jHmrCi}(Jb}C^~?HE z!=&q))E~{FS-^g z`8w=VuVx-7rM$fdaiorg_FezA#tFVy2kUBxR*o>W*kO75^znKZTBSf^6c!%thf0jl zM@L37Hf?V@j3bAm&5~p%OHCs;w6y58JGX9QmF|1q)qKsS6tDd1`p$u;bzLk~nHo$; z9h0e4rL>kdRcp#8T`OvOC+D(?Uz#_}H@P=r%l5qF@8Lx6ztwh+yhB-kJqwH{JxS0p z68U>k5?O4f7 zf=*^vcgOnafxrAPd-nf|n05C5IY`$@#(|mu>GS+U_TYR|%HuvBS7{l)O0QD`#GvlF z?2JotQd)fd)BrS^*pD;K+c({1r+6Zx=Ew>$lcp#eKcv?bXjawANt?@zT=VT^`tQ`! zcw447Y5iE^^0CaQith2|ETnICIG0yTC7#Vi``-S3lI>-xJdHM$)Pqy9H(~7Jy^2Xd zi<=kT{FM;)rhPJIJjtlrg$X#XEfNl-JvM|@|H z?s)0hL0L^5mCG)F*{XV; z_9-;ySrOyBr|@;=#d$5W&;)lS4$V7eOcrX!nf2gvR|l!n>_WoXM*1)%Z*$2tMa~FU U|8q43uBO1%6u6oK|6dCHFWA)*QUCw| literal 0 HcmV?d00001 -- GitLab