From 76e77f7424a9fccb575b5bb41f0d536772e6ee0c Mon Sep 17 00:00:00 2001 From: Steffy-zxf <48793257+Steffy-zxf@users.noreply.github.com> Date: Tue, 24 Mar 2020 10:14:11 +0800 Subject: [PATCH] update docs (#472) * add finetune to module tutorial; add 1.6 release note --- README.md | 11 +- RELEASE.md | 15 + demo/text_classification/README.md | 6 + .../finetuned_model_to_module/__init__.py | 0 .../finetuned_model_to_module/module.py | 124 ++++++++ docs/contribution/contri_pretrained_model.md | 225 ++++++++++++-- docs/imgs/humanseg_test_res.png | Bin 0 -> 35793 bytes docs/tutorial/finetuned_model_to_module.md | 275 ++++++++++++++++++ docs/tutorial/tutorial_index.rst | 3 +- 9 files changed, 633 insertions(+), 26 deletions(-) create mode 100644 demo/text_classification/finetuned_model_to_module/__init__.py create mode 100644 demo/text_classification/finetuned_model_to_module/module.py create mode 100644 docs/imgs/humanseg_test_res.png create mode 100644 docs/tutorial/finetuned_model_to_module.md diff --git a/README.md b/README.md index 4f59e69c..82158520 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,15 @@ $ wget https://paddlehub.bj.bcebos.com/resources/test_image.jpg $ hub run ace2p --input_path test_image.jpg $ hub run deeplabv3p_xception65_humanseg --input_path test_image.jpg ``` -

- + + +

+ +

+          ace2p分割结果展示                 + humanseg分割结果展示   

PaddleHub还提供图像分类、语义模型、视频分类、图像生成、图像分割、文本审核、关键点检测等主流模型，更多模型介绍，请前往 [https://www.paddlepaddle.org.cn/hub](https://www.paddlepaddle.org.cn/hub) 查看 diff --git a/RELEASE.md b/RELEASE.md index 35f1710b..b2e177df 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,18 @@ +# `v1.6.0` + +* NLP Module全面升级，提升应用性和灵活性 + * lac、senta系列(bow、cnn、bilstm、gru、lstm)、simnet_bow、porn_detection系列(cnn、gru、lstm)升级高性能预测，性能提升高达50% + * ERNIE、BERT、RoBERTa等Transformer类语义模型新增获取预训练embedding接口get_embedding，方便接入下游任务，提升应用性 + * 新增RoBERTa通过模型结构压缩得到的3层Transformer模型[rbt3](https://www.paddlepaddle.org.cn/hubdetail?name=rbt3&en_category=SemanticModel)、[rbtl3](https://www.paddlepaddle.org.cn/hubdetail?name=rbtl3&en_category=SemanticModel) + +* Task predict接口增加高性能预测模式accelerate_mode，性能提升高达90% + +* PaddleHub Module创建流程开放，支持Fine-tune模型转化，全面提升应用性和灵活性 + * [预训练模型转化为PaddleHub Module教程](./docs/contribution/contri_pretrained_model.md) + * [Fine-tune模型转化为PaddleHub Module教程](./docs/tutorial/finetuned_model_to_module.md) + +* [PaddleHub Serving](/docs/tutorial/serving.md)优化启动方式，支持更加灵活的参数配置 + # `v1.5.4` * 修复Fine-tune中断，checkpoint文件恢复训练失败的问题 diff --git a/demo/text_classification/README.md b/demo/text_classification/README.md index 5da82c53..560feacd 100644 --- a/demo/text_classification/README.md +++ b/demo/text_classification/README.md @@ -218,3 +218,9 @@ python predict.py --checkpoint_dir $CKPT_DIR --max_seq_len 128 ## 超参优化AutoDL Finetuner PaddleHub还提供了超参优化（Hyperparameter Tuning）功能， 自动搜索最优模型超参得到更好的模型效果。详细信息参见[AutoDL Finetuner超参优化功能教程](../../docs/tutorial/autofinetune.md)。 + + +## Fine-tune之后保存的模型转化为PaddleHub Module + +代码详见[finetuned_model_to_module](./finetuned_model_to_module)文件夹下 +Fine-tune之后保存的模型转化为PaddleHub Module[教程](../../docs/tutorial/finetuned_model_to_module.md) diff --git a/demo/text_classification/finetuned_model_to_module/__init__.py b/demo/text_classification/finetuned_model_to_module/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/demo/text_classification/finetuned_model_to_module/module.py b/demo/text_classification/finetuned_model_to_module/module.py new file mode 100644 index 00000000..f79be3a2 --- /dev/null +++ b/demo/text_classification/finetuned_model_to_module/module.py @@ -0,0 +1,124 @@ +# -*- coding:utf-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Finetuning on classification task """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import numpy as np +from paddlehub.common.logger import logger +from paddlehub.module.module import moduleinfo, serving +import paddlehub as hub + + +@moduleinfo( + name="ernie_tiny_finetuned", + version="1.0.0", + summary="ERNIE tiny which was fine-tuned on the chnsenticorp dataset.", + author="anonymous", + author_email="", + type="nlp/semantic_model") +class ERNIETinyFinetuned(hub.Module): + def _initialize(self, + ckpt_dir="ckpt_chnsenticorp", + num_class=2, + max_seq_len=128, + use_gpu=False, + batch_size=1): + self.ckpt_dir = os.path.join(self.directory, ckpt_dir) + self.num_class = num_class + self.MAX_SEQ_LEN = max_seq_len + + # Load Paddlehub ERNIE Tiny pretrained model + self.module = hub.Module(name="ernie_tiny") + inputs, outputs, program = self.module.context( + trainable=True, max_seq_len=max_seq_len) + + self.vocab_path = self.module.get_vocab_path() + + # Download dataset and use accuracy as metrics + # Choose dataset: GLUE/XNLI/ChinesesGLUE/NLPCC-DBQA/LCQMC + # metric should be acc, f1 or matthews + metrics_choices = ["acc"] + + # For ernie_tiny, it use sub-word to tokenize chinese sentence + # If not ernie tiny, sp_model_path and word_dict_path should be set None + reader = hub.reader.ClassifyReader( + vocab_path=self.module.get_vocab_path(), + max_seq_len=max_seq_len, + sp_model_path=self.module.get_spm_path(), + word_dict_path=self.module.get_word_dict_path()) + + # Construct transfer learning network + # Use "pooled_output" for classification tasks on an entire sentence. + # Use "sequence_output" for token-level output. + pooled_output = outputs["pooled_output"] + + # Setup feed list for data feeder + # Must feed all the tensor of module need + feed_list = [ + inputs["input_ids"].name, + inputs["position_ids"].name, + inputs["segment_ids"].name, + inputs["input_mask"].name, + ] + + # Setup runing config for PaddleHub Finetune API + config = hub.RunConfig( + use_data_parallel=False, + use_cuda=use_gpu, + batch_size=batch_size, + checkpoint_dir=self.ckpt_dir, + strategy=hub.AdamWeightDecayStrategy()) + + # Define a classfication finetune task by PaddleHub's API + self.cls_task = hub.TextClassifierTask( + data_reader=reader, + feature=pooled_output, + feed_list=feed_list, + num_classes=self.num_class, + config=config, + metrics_choices=metrics_choices) + + def predict(self, data, return_result=False, accelerate_mode=True): + """ + Get prediction results + """ + run_states = self.cls_task.predict( + data=data, + return_result=return_result, + accelerate_mode=accelerate_mode) + return run_states + + +if __name__ == "__main__": + ernie_tiny = ERNIETinyFinetuned( + ckpt_dir="../ckpt_chnsenticorp", num_class=2) + + # Data to be prdicted + data = [["这个宾馆比较陈旧了，特价的房间也很一般。总体来说一般"], ["交通方便；环境很好；服务态度很好 房间较小"], + ["19天硬盘就罢工了~~~算上运来的一周都没用上15天~~~可就是不能换了~~~唉~~~~你说这算什么事呀~~~"]] + + index = 0 + run_states = ernie_tiny.predict(data=data) + results = [run_state.run_results for run_state in run_states] + for batch_result in results: + # get predict index + batch_result = np.argmax(batch_result, axis=2)[0] + for result in batch_result: + print("%s\tpredict=%s" % (data[index][0], result)) + index += 1 diff --git a/docs/contribution/contri_pretrained_model.md b/docs/contribution/contri_pretrained_model.md index 8c9c21d8..93c78880 100644 --- a/docs/contribution/contri_pretrained_model.md +++ b/docs/contribution/contri_pretrained_model.md @@ -1,34 +1,213 @@ -# 贡献预训练模型 +# 如何编写一个PaddleHub Module -我们非常欢迎开发者贡献预训练模型到PaddleHub中，如果你想要贡献预训练模型，请提供以下资源： +## 模型基本信息 -## 模型 +我们准备编写一个PaddleHub Module，Module的基本信息如下： +```yaml +name: senta_test +version: 1.0.0 +summary: This is a PaddleHub Module. Just for test. +author: anonymous +author_email: +type: nlp/sentiment_analysis +``` -请提供相应的网络结构和参数文件，除了PaddlePaddle的模型外，我们也支持将其他主流框架的模型转换到PaddleHub中，包括： -* tensorflow -* pytorch -* mxnet -* caffe -* onnx +**本示例代码可以参考[senta_module_sample](../../demo/senta_module_sample/)** -您可以直接使用 [**x2paddle**](https://github.com/PaddlePaddle/X2Paddle) 进行转换，也可以将相应模型提供给我们，由我们进行转换 +Module存在一个接口sentiment_classify，用于接收传入文本，并给出文本的情感倾向（正面/负面），支持python接口调用和命令行调用。 +```python +import paddlehub as hub -## 相关代码 +senta_test = hub.Module(name="senta_test") +senta_test.sentiment_classify(texts=["这部电影太差劲了"]) +``` +```cmd +hub run senta_test --input_text 这部电影太差劲了 +``` -* 支持预测的模型，请提供相应的预测脚本以及测试样例 -* 支持finetune的模型，请提供相应的finetune demo +
-## 相应的介绍资料 +## 策略 -|资料|是否必选| +为了示例代码简单起见，我们使用一个非常简单的情感判断策略，当输入文本中带有词表中指定单词时，则判断文本倾向为负向，否则为正向 + +
+ +## Module创建 + +### step 1. 创建必要的目录与文件 + +创建一个senta_test的目录，并在senta_test目录下分别创建__init__.py、module.py、processor.py、vocab.list，其中 + +|文件名|用途| |-|-| -|模型结构|√| -|预训练的数据集|√| -|模型介绍文案|√| -|源代码链接|| -|模型结构图|| -|第三方库依赖|| +|\_\_init\_\_.py|空文件| +|module.py|主模块，提供Module的实现代码| +|processor.py|辅助模块，提供词表加载的方法| +|vocab.list|存放词表| + +```cmd +➜ tree senta_test +senta_test/ +├── vocab.list +├── __init__.py +├── module.py +└── processor.py +``` +### step 2. 实现辅助模块processor + +在processor.py中实现一个load_vocab接口用于读取词表 +```python +def load_vocab(vocab_path): + with open(vocab_path) as file: + return file.read().split() +``` + +### step 3. 编写Module处理代码 + +module.py文件为Module的入口代码所在，我们需要在其中实现预测逻辑。 + +#### step 3_1. 引入必要的头文件 +```python +import argparse +import os + +import paddlehub as hub +from paddlehub.module.module import runnable, moduleinfo + +from senta_test.processor import load_vocab +``` +**NOTE:** 当引用Module中模块时，需要输入全路径，如senta_test.processor + +#### step 3_2. 定义SentaTest类 +module.py中需要有一个继承了hub.Module的类存在，该类负责实现预测逻辑，并使用moduleinfo填写基本信息。当使用hub.Module(name="senta_test")加载Module时，PaddleHub会自动创建SentaTest的对象并返回。 +```python +@moduleinfo( + name="senta_test", + version="1.0.0", + summary="This is a PaddleHub Module. Just for test.", + author="anonymous", + author_email="", + type="nlp/sentiment_analysis", +) +class SentaTest(hub.Module): + ... +``` +#### step 3_3. 执行必要的初始化 +```python +def _initialize(self): + # add arg parser + self.parser = argparse.ArgumentParser( + description="Run the senta_test module.", + prog='hub run senta_test', + usage='%(prog)s', + add_help=True) + self.parser.add_argument( + '--input_text', type=str, default=None, help="text to predict") + + # load word dict + vocab_path = os.path.join(self.directory, "vocab.list") + self.vocab = load_vocab(vocab_path) +``` +`注意`：执行类的初始化不能使用默认的__init__接口，而是应该重载实现_initialize接口。对象默认内置了directory属性，可以直接获取到Module所在路径 +#### step 3_4. 完善预测逻辑 +```python +def sentiment_classify(self, texts): + results = [] + for text in texts: + sentiment = "positive" + for word in self.vocab: + if word in text: + sentiment = "negative" + break + results.append({"text":text, "sentiment":sentiment}) + + return results +``` +#### step 3_5. 支持命令行调用 +如果希望Module可以支持命令行调用，则需要提供一个经过runnable修饰的接口，接口负责解析传入数据并进行预测，将结果返回。 + +如果不需要提供命令行预测功能，则可以不实现该接口，PaddleHub在用命令行执行时，会自动发现该Module不支持命令行方式，并给出提示。 +```python +@runnable +def run_cmd(self, argvs): + args = self.parser.parse_args(argvs) + texts = [args.input_text] + return self.sentiment_classify(texts) +``` +#### step 3_6. 支持serving调用 + +如果希望Module可以支持PaddleHub Serving部署预测服务，则需要提供一个经过serving修饰的接口，接口负责解析传入数据并进行预测，将结果返回。 + +如果不需要提供PaddleHub Serving部署预测服务，则可以不需要加上serving修饰。 + +```python +@serving +def sentiment_classify(self, texts): + results = [] + for text in texts: + sentiment = "positive" + for word in self.vocab: + if word in text: + sentiment = "negative" + break + results.append({"text":text, "sentiment":sentiment}) + + return results +``` + +### 完整代码 + +* [module.py](./senta_test/module.py) + +* [processor.py](./senta_test/module.py) + +
+ +## 测试步骤 + +完成Module编写后，我们可以通过以下方式测试该Module + +### 调用方法1 + +将Module安装到本机中，再通过Hub.Module(name=...)加载 +```shell +hub install senta_test +``` + +```python +import paddlehub as hub + +senta_test = hub.Module(name="senta_test") +senta_test.sentiment_classify(texts=["这部电影太差劲了"]) +``` + +### 调用方法2 + +直接通过Hub.Module(directory=...)加载 +```python +import paddlehub as hub + +senta_test = hub.Module(directory="senta_test/") +senta_test.sentiment_classify(texts=["这部电影太差劲了"]) +``` + +### 调用方法3 +将senta_test作为路径加到环境变量中，直接加载SentaTest对象 +```shell +export PYTHONPATH=senta_test:$PYTHONPATH +``` + +```python +from senta_test.module import SentaTest + +SentaTest.sentiment_classify(texts=["这部电影太差劲了"]) +``` -**NOTE:** +### 调用方法4 +将Module安装到本机中，再通过hub run运行 -* 为了保证使用体验，请确保模型在python 2.7/3.x下均可正常运行 +```shell +hub install senta_test +hub run senta_test --input_text "这部电影太差劲了" +``` diff --git a/docs/imgs/humanseg_test_res.png b/docs/imgs/humanseg_test_res.png new file mode 100644 index 0000000000000000000000000000000000000000..d6f689f7934ac3f702f18bb4f7fcf99c21325de9 GIT binary patch literal 35793 zcmeFYWmj8Wv@jYVxCSdOK}vC2+}+(9XpvC7I25~Jz7+jA$;!%FNvc_#xLM`MHZd2}*84W{;&Rfk_k^?Vz)yh_+O_fj-~PWO zf!u4J?gNnGd1Nd549(DafoT4lB`%60^0c)Mu(fx#jQo6;uLZ@R&bEDN^#aBFe8|XX zU&}QEz~A9S;VgRBal5geqwsWcnf<-V?WogB>s!*X?Q7nBNUFG7=X$bKtn_&^5yHOiXdBWm%xsJ|XtBvet~f_V9wsDJUEy zbh;dDVBbQ3>oI?SAMErslf+rKWmO+qnwV-_+O4H{PIbJ$lzrAMH{!jIxyHSh;f#`X zJzThzY;fAHmHnXIyv9(EuoW{crd+l3~CT*B4GF?&Z8u#A|@AGAS z&o*@0S8U_ydq#V!AMMqn*Xj-O^>~JnP*V2%TszZ!+PC=(+r2O{JQpHJ&}t(HnfM5Z z`p6OxRslnUsI%j-YdL0bUJoAY+2eQ^ zRBY4fdj_O*IW0p)&(CAlZ;hUuGCUUDnlCd?cMkuyJkZX4x-H>TRaI##U9iofGYyql z4@WboG%FLQsQ}|Cv?3W}q65g@RPOc8idHRH+ES6Q&MNkF22BpyF4UqI+AamcX$&n` ztAYkQ2JY0$HUA54`_U)78&`iGLVmuEnq}m@*R*zDD&lq9Rt6~(DOk0PG_VnYvi9_{ z;+N1jEBYdam7;;fffZpuypP!9_W-IH+g`dJO_NhK_6FOVtEoDZrCIFy%h45<=UGaY zFavzS6J8GAiQxGPH*8`9NMbrT})w{v86D)xMc2ddOz#q z*A(gB06b7C)AJ}zAX5SBhcVD4LxJhp$xr*%t&}mZlc}7V7!L1_B=L;WdbA)M@*YG^|u3l^#`^WvUzx;Paaw1r`es|v;uV?1UuH-iK5i*PgqSHAHuN;q- zeaP7#mghL%3)ef2Wj)(xX8!at$y+d_?2$FF5g<_vUn?`Eu0bfP=m5z?C3jiW~|}B2q#On2l_|U1td_Cy`m}5?`k+BuXXm zm&bQ4)?JMV4^KnL*C2K7o8(j)`B14zraQ}Pt&-uoOR5(Wg~Ml8f6sWd2>VtbM%L%p zjh?(!00^R%Brmx>*uFG68#dx0 zuWa5QDes$Wy%|Y*ai{1*ucLRNATnzHN8tW~X%xLDcACcVvg6w3s|SND;mLx2AUn18 z%D)S3HYN1Nm06P-g}SG2IN~ex5Kl(3{bp7f);^>$7YFuYv_MheAIF?Db-{%2$Kx{HdV zs@(Ex0qb&>4ADPM6Tv0~qu&NQ2E9N$FY$8SDk&lvB@EQ3OH4+KGopjn$K}*O!P807 z3+V=!6I&OB&<3F)wnGB&y7U$BF9Bh+5i{Oa6?ZXyx8Ll2POs;-%`Q%Rrcpc_H-v22%XKhcJCF=d1vbRsy)hfi}p} znh|+-3Tw|N3~xq8hfnzTBCB;D=$t6ty2I7Gz&s$56#M&~&rj&0;B%qzmg-#$u#iNTV+{o;8bI|2|m+3R7J z;w{O4f(tf9u3L7GV&+H;O&?+WP6A;$7f}&VGojf_C={?lqxTPL$Jl&_+=F$}A7OE= zGhk<|$!Xo)3BiQ235RQ#0+Piu@*JUDv6#QY94ZXI(<0fhmGWe9O<0qwc=mR0DEi4p zZ+$0V?E@9hk=O74HfUywKt*6e{3TEfBOBa{y%pvJvBuK(2TLag4tfI9WD1CKrzZlD zBrYguyE`pqiDx*8ZA|YLLr>AY%PKtU!{nu{9V6?l2S_k&6m8crKc=v$G&p9)-9uas~J-OuG@6uon$oW_a5O8C1gV3 z;O0{VKqOZY4|sU)XKERr209&gm%ccVQG#|4>OvD5=j|`f5{XO9C14~|0LrlghXELC zQ`53!8k#F6spGcz|1;at)-b_DQzBj|Z2vG4QF>)hw=Lx?^Y(YM{Zw4x zyNC&ZI6d!~i)&Yit7FTA;eJfNV}?WE4nMfHJV&Ib_4HjCQK`yelCuP$gnLjUa>0ox ziZfnvweO!L4p0Jb+c-+1z)+TsJ2gjrFzO@y4=IU)S!wn&5Ky65Y8~WsZ^&Xm-KTr9 z$QQ2I67Fgf0t%pkTV!B>MPHvBvqgq)V_ItJ=l|VQ*Ay7==M6?fxTNTt8Ee<1J3X!& zl@X|^C)J@IE_7>=D-11hR$Z|J11}VD8 zpA(Bq%t)?)l(v7*m?2OOgNi`7tiPLU7Wa+g!2u|d#Z}wJwUm-NkWs=mJdmb%zseK= zjSx88E}laJ*CN9@O8QK56&@pGUW4+EqBLG-)N6MbVTTODR{wM88iZR1}#qM5T z+;+`5Y;Aw0XHFGl#N=oq%#!GX?jTSX&QP8|z%mk`tQnT~2z@?{RyB#j4{^R|?9Ex9 z1SXU%5-i~$4d4hyRv*AtlUNO7nKS;DkRca_$_YoRrK8c4sKmlrovUTV9HGJBN(T+<( zoZ&=`E1rvVJULJ=Dh9tcR)u;A^!W}=5kEgln79+q27SjGN$Q>gBP*~E1BxWmy+upp z(_^xIZ(C|VNTg3tekHffuUbD#(FgK)CVwR>N$fQGbjCUqTjHau<-v&c2GbY;BTKg9 zQoV$y`%-(-DPbrtcC2fTYIl$)?dPX<{bE2B@hk}w$d-y+wtBTYr`RKr1zY=%Yg#$) z7pK~Y;)VAZ0ljzQ0PlkKh+nB(5GG?9YIV`GX2YD=>ZJE%jy3-m_5XYhE+{M26S-Q@ z#q&~4Kjr=Y{#2Q{xvBdw3H!FXRZwbrIYMu|qIC)Vd2Wu>cYD zTAWH*@VqwyI}Dwbcq{LF|Gl6i!xt3&E=WhGH!>3B$0(|X)r{qJ>6$mbDOC4i%aP*k zf4&ORfo*WdNw>h@b_*91*bz zcyfT;>P+qUoO)#@zYROCypgYEgPK!9`ywkqVCxV(XjlMuwoLc%fH5UnDUUkVxsz|H zB9hSc`Z&-Dnfu^x(wSL@jDB>UY@JaAjf@j^g}y0Oy55Ta9QEK|umO^ITX{`G!` zG^d6`U9tOjNlmRUL{9^+9?yz?-2Yd8HK1j|H;BK~(^;diK7nRSVu<55%(k0@)!%O{ zkbGx~7>|^!0J{gE&PH}@AS5~=lyVvC;;~rrf|;8Hsgp3(h zC%@!}XuNd(s8}?0arQmBQ;+a3Rg<}Czl0_Ku1k1e5c}hGM9}-{fd`7b;*#x)8F|6n zCO)q}p@rGbUM3@+!uz8+NiX_vp@2tqO~EjfmwqPg%CkfLZR-H;goZ*ys0xdQU0JZW zv2wistuVF?YkCw~B?6fVYm_XSrg-e7LrMpV#^qpT_(ApFj{>jNokL;j#PjSE|SHONU-4)sLt7JK9P& z=(1%-5AP^QeE4{OLV5=~SL zDX!8~hjeiESsVX5c z=QA=u6N@(N3lH@+rjen~?n0aUFzO2qm;YSzM!DVt)S(o>-U-Kv{eI189X-Ymn976k zy(?kF`YV$iFO@u)0i-~NKx>TWtwx)T8++-^WRAu_hXOMnAQC1FysWoljbU9v6zkW3oaWRevKNozf`_)-1^RPfg<-+;IL8|%gF9u5qe~I8+Q+1zg1%3 z{jlj0Nc%6|?gI{fH)1tCZrEE*W5M=b^0zRxDGeaXsmTll6c8nvTgJqWp~jExZ4`Df z>7z+Y2i~A@2DD4!kS{9u?kaiX`D+OW}V3b%eD8(CEIKo3ZJij_zFw zVIm>T>y`h#GH@&9{103GQW+u_Hx%ek{oGGVpv=&fOQz#rbf)Q9iU^rb)&rS08ZoBWa#eZ|j_ z@HkF>lA6*Tqo-Z%X(=gi76T}4lBdCvjKWoC&nNm)e)@U8?nvJRZn{p0sbA`DH?HlU zw7ie3aWJ*V8}5EIPe zym8pknOj#fx))?*WIX?VYDD3Eu|>VaO?ZZE;&D=dFKbaXFZ^-i2$$u3AQNxc*Ue31 zQ%rDxW+csc|09(Xq+<$WSi!7k7!Obh%h5TQyWcqX6TZ%gd3tyevitg1WQUN1shzAM zyN|l{NlouR8!ZP%H2G`-_nE--=XRRB&C|cWlywWSM**B8vG&)~*H){eLwE$eo0HR_ z6?VV8cr-0Fwpw#AOfduj1TZo({Fk-QH}Z!(~DYaj~&0OM;Bk4#vZ!{gLhw|N*!n#2tzC*;al@0zd2cnF|Q8b@Y^}W16|AcJ)!1JO;s26Xt=z5 z5^@u^HTOJG4k#tC35^KAp>phH{B)6oB2LAy?ahFeVPni4rx(9GO{jTaa2P!<@+pfQnc&7@vb&Q}-KNT! z*x6A9n<9+e9q;|U>N}{9aR_w9zc6~e*SO0Hohi}Vil;UTr2ce4VCXT!K?}Zmxow?l zd(q?D6;X@PD}iYhT9}WuJ>LjXYnRWd)y2AMP+&7@iXyPem;RyhJV1f_c&1_E1R%P1 z7&_AG8PXpC_@qM_B>0p{ZWub&!K?}}9Y+01E<8F(Wkh-9;p^C|r-Hhp1*zbiLsg-# zOGJ>8!%@-0qfho8henBFXRY71#@4R<&acb8QO0~ve&{|x-Ih`LI|Re;o@BB+W#0Hu z6=wSivOnLCIW82WY>AwD_O+|M-cDZcD1KF%$Eq6}8MFV(mrjm0PuOz~A^2Iz6s@Qm zDJN~V#ORY47MX;d#~^$WsdAPUTmMNTzR)xT90KP99|ip+LViHTv;aLwKa{~~2)uwn zU#n0coDi$~{temXccT?jHtw%0Q`T8wZpac2$068#B%4&=N#Y!bjrr|rv?oATH3BxOVp2YuX!_vC9a(~w+a3YcfOP^_dVa!7`>ny16@5AG_w%hw}$Mg5U2a40)q`V=fqLi7?#U!mZ#ndgO?yF=( zL`p7&(SxN$F!;hXt?++Aa4X$hNWR|#(uu-S2^qc$W!xLWnimropt&ZAvN{7!v1tT- zns`jdeYAL~`n%*@e-|ooF>`3eX~6VgMQNwr)boMjeP{c&=f9!er%h`RzZZ|aMNSMG zi%cKL-9<_ZV+~dB~9D{#ZM~edJ`Qh7_ocs_c^WbONJI# z#w9IHERm)a`;T~oN#UgY6Rd&DG8*z7WHUr!@_Bxg_~vA7?6b?I+x6&ykMfIbD-DrA znqft!2E(IwuwSr6mI^+JZ9zcWnaF8~fn5vq1Obhj0(F_W{wX0(Kz_1fh8t_;iRGHry^n^bChNyd zoBm)qS>&jrNDTob4vX0`{CoFE3J@M6P_v;0D-DCBt?`a@6kiRETY2Ae-+dUnJb2@` zHB^69Wwd_(rq#WGBVo&K9ay4v->cW=Zy~yOPXiYI7qBPh|hyonD52MQCz2pD@QNguFn;Nn$-v@ET`=9FCUn~$eNVjq_ILz4g-M0 zzG5Z3u*)Zh0{sbxa_$9v9haz%(%u#FIa%LW80-OlF+he{Ay)Da)({(7b>(hXK$BCz zZE&C1$wc=3wx6H*+@9F&J7r~6_>qrx6J&dLC>p)*(^z-X_Zh@AIz#l{rlaot!0x`- z51V^Az=p|gy{qJ6U`dX9L9$1Wr}tnIpX?7!M$`R^O>Y0J>P7J{t(wge6X9uj9#He$T&;T@Sv!5)tz+-S^}Qp27rUZZdL@MRx~+Iz1R3H7sytsx$$s`O9#F@dX$T6;W$ zsReNMKGm)&Py+;#-0bHx9ER4%%E9xiJGo>gL5d>(d3m0Mw6)><1z!ch`pn6hfl1G? zEpN?Hg5E9hDS>F=l~Z zf@vfxW~;c&6T|%8qB-rUjE$OFdzu$;WNxAHcN~0ZeJB{)n9P2qIQ2ReYH7IWgs1C6 zs~~@r-A@znKGBCaN9G^%{pw|^|h<7ayAbs*na_%oIFSTCru z%-YKa*sJJxoqk;y5yqa3!QyjIA=i$so2|i0c$P=~3Y^lKS}KDT=0e}r&%OWaY&GLy za62}%Q(TqZL~zk^P;Uznl9^`a`$f0YQFI123@i^ z&rrzyfLoYINLfVo)4wH&50R>Ty~GoAJ5!W8Us~MvTjXC@PkeinVxYBlr>tD^Rt4pr z=*t^URzuINM;Am2;W7~Ln#Lr8xcJC#(yTQ0l;6KIKCi4f2F_RvkBn~oDJf^`(Wn_& zoUg_waF|D9EOxB|qDisF|D$IKJcS%kM(^a-{oT76tiJVp0TESu5#&4zDyXD|A<$p1 z9bDJfFoPp9-;-aObKb8C5Q5fuveFWJGVpe`SOrl7rGPehd8-srGQ-@5fzaXJT zkIBHR&xS!Wa^6=`uR)%90Zh@DV9kFxn#oFy&VpQllNEG_llz8)*O1z4L0s)j) zN6OYqIGosMHf+|{g2HPLM zV~j>EZ`>U0?p<(f>p|D0&Pfd9;*#!0?Y1eLNLYD)lB$UGtM7f;ie4(~R@EN#Jl&D> zFzu-I1bW*@T~bP)Dh8q(&$XB{O4;^D*q!BE$F^)Jw+Q$r#Ba zSX-OarWmE`DQdAlSxSef^o=Cg%+XIdCOLD|aAw0v>(h(Q4>$DcISW}sGBj(kmwi#! z9$3Q5xH`HN62p5&Ge&V1$#tY9RnlbC%r=}TSuD^dCwKj}g^<;00LF)Jd$pc>)2XD6 zB{$`Nv)A^TtW>T5dI-y z^-|c9aUio-RsP81IU|3~Ectckr*sf$&fbJK41`*FUU8gQeFrc2SMw0}c3Vy#rlx&r zXAo;rEnQ)|&*)t@7H`OSyA!O=<*b{!c%H6M;NxcC)&wKgAiR89mc#+*FzcjrH&SoG zAvsIze0fH#u=ZxPNEqM)EP512t||#yU%!CRr58<;7#jdrs|dmrLpac*iN2ISE6YwZR+VLpAnvdSzx-%mRePlr2$r-{yU{o18ss z+SV^|<46Q98Y|M5&dyFnBiTL^hrk)RFg(V$h3i)3;?h(fwD$1A=@LP#4=do6ABX=2 z4zKHXz=b;(hAW`e+$|8m#GOimeGl1@zBTOMiq{R(*qk;KVUc=2^g7)UA8iM7Z!k&E zPz7_wdJ~CO{bLn#N#8-_pZGcHL$6v7eWvXZDe8w`f`E_J=GXu4_M9x->9oDQy%AYA z&FM8>TglbJj0%ISLMy@vj|esi1H;mYXmpoh`@)~UQEJz{)!934QNA4~ZKciB=^xfG zdp7TKx0x%ODA6t3gxjzL7@`eZut`EHk@wnLMh2j1<$5=1Ny`W3WXN~+#XUy(TB0So z9>s!qf#;g31L>espQW9OlQFpYq3ecdzadsH(M6TXcm0~;jH)tJ9(yzG+7JOQn>=J@ zIbP=kEor_LVa2Z*l?a_w3@U7Upo%k9>K^+q7AY8JZEAItf>&lj` z-4<;d=`g7N@f(B-O9zM8kPs9PeJ)ZP$cBzkgj9*Cn1>1t!KRE9w|;R)&W5=3*-ZK8 zn#q(2M-sTMK&O744|RNowe{vm5fqqzFtjP%X=YrgF(=Bb@`aU>vYXJ9Z#}e{(RWl%ll)QYvREX_75@ypc(|ITn%uqIt_!{<=fJ zC4rXlym3OfZP|7f*)q_o(wguHxWtWeboVZk;wX zl6hglNQ+&~z>$b=#Yc6l>7CrDBO`YaOZ~MXJ*W15)g7YSoh6O*4 zS~s+myk*)ZT5-9g@VRQUI}aKV+dKMj7kO+Di-{s*!%}hS^uUW-Xp3H;$ydMen{>5_ zV|(=F-z>UT)ma|HEo`hOVkVT4an@(gzzAb^ch82!hmCRwu5;8#k|_Q*?^+8VTd4^w zrCN@3jV*UC%|^!)Ba{Xl$EKUjaag@mWt^p#QcTh>CF|cf#}&pN#+X}cRqBSQvv36U zAb;Yoa+ZR$EIhPS`_Diy3$zbOBDaMV8K`^l_~<6`h1KZQW*Xx?DRql8hDUYTCVnZ8 zsSb5d(-SjegMZ8Tp0Z6G@ptVQQg!PS1)Hy|BB7$DM~$P;K9kJtFQ9sFcfCd!tvS%G zUZ1qFoGkgRBcva#r&l82D>P3tT=P}%ro}_cWNfk0HNs@)rAcJMa``KqyK7t~v*#a~Pb)Bx{dJYQDC_Q@>(e?a*7;&(s*=Qa zM2_9|YHn_A<6{rPUJQZ+6-d#1vOF6|J%0ALq?=zMl8h6TWL9CH$T2RmgnzluaW#vN z0_D&;xYW0u?CDgEqicM8{S00iWV4aO445S|#kbj>M^r4h)&@Y52PmQY;e<>p=!7*DaB5ZA(_~GjKB$1E4Etwf04hdpx@rsCunPeX@WXit;zW?E5Rt{-soH+DJ#g!z|ET6Dgqp<8VSt1zqy&C{NT!IX zJyF$S3f3P|@KR=#6cj1q!4F+8DbKjbm6Zlo1?Y0C57Qc{`@at}p;eYvv@{maGql18 zk~U`8vtmXOMO-6TyEK?EBf&IP|}R2y;i{vwr?*P+jBAGYL%My7ak%SE6? z$=Fus6z%IlywmbKfzlYfKmJSxvVj$p8YVqC6o9H+QiVcES(<3V!IJArqjUXU%3D$b zRLu8+<5b>d9r^9uX5FSatNGy#%s8rJL)}k-+EQH+XdW?@KF`(}jlNjW@5tNEqBXNL zfL=sLR-ar`D)JG~N#gSotVC!zM$lV@QF4npk5~{r4CGYP93&!UbVE zV5{_SqH3BEA+ERVI8Jd3Ya@CD@ry0ymAn}&{)>M`H5XXp= zwEY(tA>JCY2_TnK;v|tG#2jIrb~s#BX#Dyb$xWWKSZDtVh@GF~KgouhPo!L?&ZX4{ z#^y~OKhJ*!@Sevy>kkRIM4KQ`34WJ~1aEpyHCAG~Iu_IH@eV}sU@zrlH;r2)NxO^S z%V^d>|6PqJ&(~|KxAh-xQ5E(;Rr#>Y4EomUxN8N$aC-7PMd|!R*>5jHRl#ooHnO7q z=;-Tl@h#jxCIZ#yxpL9yBjQRwp?6CTXSh$lC1sDrLo_X=9E93BeBdo5`^{p%)EA1Q zj1`9FcT=71uyb;&HQgc>1TDCK=vnK$-^>0X@@*v~RQRex%cd`sPjBz{Xo@Ty2jvSh zP*EBYC<9~+F%2o?uDSV@7^sN>DcN?@9fx$m^E@fObH}HxdvT{8XEms6Q}Vv5S>-*y))G;4GRww@Sp(E z^x|V0L!}zpr9{t2Dy$AfK82X;m{IoV$3r@ln76yWGvGP%bJaPlkm7o`kGY-BTeV$} zz*C!FT(8T zDNaGK6zLdL4`G6FwO*E{f3k#|eC*AuhZA)NEi_1A;Ch!JAN;QY;*%yb!$4hTmC>Kw zZOkh^(~5K;eui1J8FCfC0?G>{p8$1C9*Uax0xS3$hepq>P=YeAp2%K3p`s z0#QAET=`(ITT!!%0c;G$Cy>IDLjDY{vZ7xQKwpgbv^`0)pUgnep})aTTqd%EqOlzF zpR*H-_RAmB=%KIr^!XYl_I4Mu8=SHo98bpb+cqwdD3I~{yDC$~(Fm6Cggvv|sg1bN zvaj+qpT05V=%%MAV(2U>c&jtJ1N`$lS8AJ!i->ky?C|m{iGjqtM2z`mV%O#bQ5E5` zdFYJBEAyQC2(5Ah@jCCAbbnkJ1y1L+GSn=4aoK zh&2oYiY+{X)=cjN7nvOoW7)&z2C0myIsVCYy@RKYOLB{>>SH-_RloKzj+YAyJaUV9 z2frS;Ec6Z;U84|2Ah};5kELNpNVNgIu36`TSm(@yR`^JPA^n^3gCv*)eG0E8CDa-K z8kx{RuydvkC&|C?Sl0si(O=(=U-*WuwYD1(= zlx?B1L;myYr{uoL#QaQ@1h=(Kri@o&slxM& zvUw3vh1oc5>dhu6<1Z|SBL{By=O2c)Y|&@-YsYQKRO@cNhE_y=?x!4$=TvKhGX*C+ zqec<%(&~FEW%5W(Yny2<1-9d23ENa zSg+|+JVOcjnk6?D&$Y%f(&~?8g>O4k%C2^tDNz2)#eTl>`;0WYa~9s_4)rj;KAv>D{{E+Z6 zlPwu<-h&ldbV?4IoHunDd8hw`EBdP06qg>8$j8x-7V}iXMDnLG@JJ;JbmbZhWnqx- zdtM}Dtl%>j;xrhFT#Ym*5#6tE_$^)y3z_|6qaqT5ZUR$=6b1-3UH6@t9qsGb(87!? zegpoKIk=hCVgLkdjHWzlqJgv8=Sc(j{Kt zV!)pK(H9M)`&~Npn`f0uBspr*twdh0v!b`tEKmucKlg09KREdZ_Q*0tFrEcxp%F)* zp@Ap$|S|#^X1L0tX3$ywnm2DFo1v+l&%pp-wV)K#4Ji6rpR}N&@kD} znSb1$zdtN8eS;9G-(4v~XfS=?3cpAQy5`7`f)SkpveDv_bW89`dKzkuDioH~7t1pu za6V`mw2yCq7fC$|c)sjc<|nx$$_>0wn=X;U;iyDVAx)t-hNw)yNPWQSn-oyNdD zM}g@7>Sn1-(GpT66^9mUKPgt+9{O^vzj{%OibY1A_v6bD7X{O@VtSVu%NWON7=?!t zvl40h79RFb24i)B^pq&8thq~Fag$BRPHy6}Mib@kS3%PZp7*b&7d%KO03bcewm zgT!CDiH}`;O)VA8XyS+Em4R#{nOt>G4dRm**NTNcFek_TXrv&*FxR6uD0b}FF`wF= zH)4=WS-$Xt9PF@thWT1ArG%dcVx$~bP$IMZURocnj;NqtSZ%qTqxwCQ`oTckjHBGR z;8+zU9B^e=q(rK(4Da*8B7FcAxpo($^$7gJyEYHvjO>tRBJz&JN#ZnQ&Pg1jmH$Lm zqiH4MlgwnVLs5gJ$iOBH3lD=Y6c$TUqCWQi(|-QHXlri;`=O`9uhxPjov z`;0~XPmA+0GD*`(19Z`9Li)&*Y(zEPS!!_cJ6 zz_-n4^x5f+hjF*hUlf;BHr|Cz_ELXxotl4F_-a+J{_@ecOZRCN)2MMWWx$c}KVbcF zoH(Nx?OLw-O!FM(x>ed%ulwg(t~Xcs4842{G`&Wm2Nu=@T`U=Cz_qh@3*D63?-1)( zfRyAcR%ud#XWAx3>g{*^0NKz8682sAh(=hNcrRi_SlmNhVN>jMm9CxM&)S1#%{csD zE7l1o@(`L24@l@=K>C_G*(V(4y2BCp8`_CU5c)Pc(kj~iouS}Loi$Pt>NZhUV!XbQ zq*m$Zylxr&x1iU%I=_G-kd#gEZxY*;+Md^eF?PS>TfeidIUVbnYd|sU8XY4~=hiIn z4hN}O+c3V@gcBqz1CL{{Qk+OAna2MagQS8HQJfH*vH}Z9AQ~mfKZJ1K{Xi{^NYah_ zqmaj$YbK`&di#O6IwM7;EWNBP@SVDq<+FeMQLoV^ zqDLtF+G7vdSK8uNGz@aZKTmG+eG6DWw)bNJMy9-MWCz9g96?X4Lrj#d7{1I-H(TDl zt01F!420m*N)!~iPRUjOKasj6#7u=9~W0#3jeGpbv zHmk~Nqo1_C|In2`;PA7>ym48Hd1ES5DNr9R61fMF#HNP~vx+#NFb;}~tY#}Jrp;Ei zIJaxxiecI&%0B(+#?8}m5{7@4UtG>D6 zvS(dq>3G)uW+`k6i@!eM!t%eDCAfs4+$fKTbJ3m&6C&aUd*j^G#C90^Yp%ivW0)VP z;G;zdt(pUzKe+iA;EirHs+am-LEpP&<461}!sD}3&T|mB(^UvBJ;Z75&$$dJNG!L7 z{npm+=3|sfW_^#o8NIxnbIb`VnrO!~x)@AqrHF2Vmq!y46$?UM@3UVu{Vw@V1#y3E zxL8X;zJD=DI?DV`vp949L#VMTS537uzb2EDbd=eQ;Ee_%S#Pd}jp?6BK&(3=GoBSA za1rT)1WQOcfgFl?peB$--tD0ZQqClqs6CMY3aHf82jb3Gfl5L`;a!eT2u5Mr&NCq+ zO?ugl*K)%yj_i8PPA}^(Y2}7j3f&q(Z(63qXTn3^pv8V`d9*(-IkfMF&zEh{LWey7 z7nxB#;S5fNObkjjqq=E!f_8bMOJNN=S82zC~*O&hfKx6zGT>yb43YIB~CImk8oAS5Hgh^>H$nT`Ow{GQ zW#G~!c-+ogH651FtT)6PLQ2zV2n!epW=yRKo0P@I{oHnqu;=Y#=Vq?jOd$_cM&a;S zDA9Ys=Rcu^cRNX%LLFv!+Pd?o?$5r?F3wEnFfT+K?&+5XMayoE8Z-fmQ;?PS^2rX| zGvsv-#?ygY-H|axJ5&?a0}9#YiMijMG2ioev5d}FWgTr}1z*C}pX9_s zq@z#_$-m#^_N1Z?>v*+U)vo`${>}xW z9X#{sT>vql120!8+Oc)LU;OR9FjanoJA)?xMo)7INuHc7Q^AKtZX2*KTx|Uh~ zsDEyrt_oPvpy<;O78MbxyYEXe3~c5A##Wh13II1(TD6 z0gd3Xp`hPs{?ezKxnt6We-*<-E)&oRTEl@ngJa&7$j7S8q1q)&A>5GsG_Q))BKtQW zJKj$LmSXF0UGP0jy<`?I^qdALr2t-H1?Bem?pvx%Td}?IVwl6d^xA{OOj->AL*|F^ zD|~!^M6(TSB{zl-#`iHg;}D@K;{|dt_C?fv_L&(&G*5Z{hniuW8PpvYFxyy^G{|66 zBpsD(u=}OXqDx=^FwSc(XR z=oNnvDzcFhBZ5U-yJY)BP&$zgPj zpq-IRM1!}ezqrlu%k(3{8B<#eYMwbp+d?N1qW zT~Z(Y8JeP3l;*n?Iy=b{_+}p<0_B7MLuwR+`u*LXTwGi*W~?ql8w>WPSG3??16+#( z=O+(;c6z(1t!mf79dsWJSI136oa^VTu^qo+m*Qn*XH#corL*dk=y<tMP7d@Svwn8OJR9M;!L7$z`2|)vS{5bZqCu3{<>J=iXY?_nCpM>u>^`c6 z<1ipwgIJmb?nZEMaGVq>F81k4-JAU7sx#?6I4^|gci^^tbHtO~@C2O;o*LM_7+LK8 z`3h|wK7Vs>?wZJTB|ra7D)Hl@x|-uq^jZtW5pl1JEOVInjP+OJ*8n8CIYVc*QgUv` z&Mi1}d8D%QEna>DWFIZcblN8c|D5cQ(#Sh4)V7cujVFF?3IZpdFqII3fjJjD(yRe71W+yM>{t=-^_jy zb@9GuoVp^ha5N0EwwHw{{T9!e14Y>T^vE&6CmbVQ&fFDII+0V<#&1~gn1a#Am<;MD zC(5FfUw3ITRCi7ByVpizbcB)0FeBp+Z1(5RpGI}izNpVF6Igl=Wc(+T?45$)MI1qQ zcYeNzLT*RG1-d0sX0MYeEW^O6-n|}=yzRd(ocr;6`APf#{>Yx5P)j!v$(dQDN$z=2 z>Z&)5ig=9-A9v`N9Kv=FvR|JfmY)ng3OEA(@0kaNO&F$L@9q(gR`n)BkQgew*>v8s zw6Zh~q$dcGCpBH@EaWt%IQ{r`T_ImeoP9W{n{jkA3T3O)9)19)wVMz|pRTslaqH`wtFYk!CT4;r`)z{kM9wLwfi)y4M)h(LBuAwLo(L z>~CE+M4^fn|Dn zWgM;CYK7}<=0v>@e;OI=D{a)jfj4q0;Mr6_=TxVyVb{H>I{%7X4kG_H=0s}-T}@=F ziTufI`h2a~a$?{#Dp$u+U!&3ocKkm?y>(br-}eSOAW92Ei8KQuB_Z84fPhFy4oG)O zcZW2BG|~+-bVwtOba$7agw)X7!{>MJ_dXB&yU&@k_gZVOwcht#oTkr$`$;iN>tGHK zKcyl^mK`=8FDn6cy_#*;E<-~jb}_l`m0WCsOK(|xTxCsfCNerlOjjKhn!0B646k%G zEBag?+Dt6x&h^kX$c-b=pHujUu~3P`IRv`If0a@3JZifu3RX!_aJ?d_v7uhiwL0&? zJpb$PrKV6I-sL^gntDXltZ^6VUU5*s68@T_=gAFHk_q|R53*UlBG+Lqdd10_!1Ckw zWEQIi0GLJ8t;AEcP{rhcJXcoA)_>ewY&*XHvS|^j<4n~)>m=lbllWo3syia4vlxy3 z`STdU7q7KmqW~20E)~Qy2d995S=9Xdy{XW{oprmUKZ`(5`7kS)unpKY2PZz3rNvQM z)we&t4{_BRsn=<+gk$6~!crsG_!)q2i}_X|)ezJ9e34(v`4s&G%4>z3pWmNIG_9*aA$Jvx9kxGDjN5coRwVv&! zWU=ExRfs^S*0H@^pJJ@c%e^e;6}Dn(|1^9@PLC5MTnO4*KQ*#lD8d-ocDgdl8w!zh zC}97xskg?wtc0q?^jC;KU|?_0&=@{zC;Z{?h$Nlv;IE75u7!YV&Tq#90a>phy>xU4 zss>JnYSmY45|UE*91y(U0yc|T%qJO!q4fSkMRYVKP57XC1C@#T-j{UMNyQ3SS(mhi zELq2b2YmZCTfpx-ihwEx+hC1^(Oor)zCocUf*&v*f5p(y@cL=i&5=QyuAX_1)m{qP z9Hk;Z|0@8ZBB^0XQL91HCY2N@}Wv zhi{uj)F*VXDdN*iXoV){?>NufCi3rIQ7eLfuUy(SD!jFrg$FZ_HdY%=xfubn1lDZ_ zHo2l}po`Ha1TMPEqJPW1v-7K1&6*BWL5MH)qwXMU+&a5!eZlOxW^&z~K!D z2CmDkoJeo^ir^V5I3OiV#(a0z(N5ak2xwc5TJn<5HCp3!a=`4PK22vd{z}r$7$1y- z8ONBMqOy{-(!pk&TkVjXv4!xQyIa>J<{P`HZCEQWuTCb5^vqiG>h*-YK1rWQHvrQ5 z$c(#gt$#yy@prztWZVMBgqQP+LMgKJ^QhgC@9p^1XBV-%XU^ek?h~)+Puqo#G({ z2jgC?y!eG}e0sWV`vAZYyU1ph+;ca8AJhn(Z(0bq7+)4T)Re9XBw&+cR|1Nxoiyni z8yFZAut#rFt?gG{Q*eUY&bCJ-surqj7klMft2nrY_+$afH2^6Yvj`2#E9PmHp(o5T ziFyH+fIWOtfEvkt<9vnsLsujPr&z;PbM`IE8@Y0`&mSZ7xEbGfjU6{=YE|nPt$0)95{!N^QNuV-Jg9q9rk2G4 z9K}XIO$3Cy3*Ui=E(SNM3M5ELjMht9rGYQ|>|BP>=;;hYZ1g7SyJWBa(85Ab$9&PA zym@Wix9+)4Et;ObLujkB)t=K<%^s823#sPcJ$Ek-ik5DNibPdbF6$f0|F?&uw;8*e zPae+Ly5$vYj8IsadiMOwOWch#|3=yva z(oO`C_iB+56T6hu6RmdNR*$nhD)mndp63&J5d#5|fP z$Zs_!_`8W#)L-9m2p*hj8?L!jsx;QTsG!A(TV`hXTv43L?`h=GdPtujL@b7lY*Adj zNmWb$(q@QZFe5LA13-Y!?WeDxT`G6Zc0RVWe)-xUkOL#I`!FS?14~(6!aBp7DRkHJ z($}A6>{vMTLYRr@gpnlD%)~waEw@$^+wnO%BD~#Wtpf^AaO`6Ip#11F84=k;k%>^O zurSXl`8TvL|MXRoC{K}aGXzSTiG7C(N{j^YK!9(S9lz+^vGpd36JO3IGLKP`s88g8w4&`KSFNh#ED*}%zcfjG&PZCGGaz66nFJJy15j`?W$@kA7Rk3`a(T%#@9Zz{C+2G{tXJzg6P40CW0buV-_!AM5x z-)yJ{Mh*igpbvsxDHE1s`1FGuiKGYGG88WvXJ~NcU?w;qyShOW>5?~`^nv;5D0_~H zp4uyC&QaoCgaMW+n8EntfngL2KOE>1xVi+_f%_VPa)}x~S{oiXE3C5HzP3&!Sq-ir z+p1Q<#6KU;?qSvB^=_at5>^-Coe2(&Gf~mp_{^-V5fnyixd&}B(xn&P!xoQO8<@Tl zDiDei@q|b-@sLZ8(>pDOOgU_90MOVGH}?CtrFG0?!-aSZu^v^6#r+e0z2SLMLa(RF zDk?N5Yc-;*>S7!lA&K(Db)KJW2tNEI!?tGahG|(FAZ(+fN8^v?A5wn25=S`g|MMK= z(KCk(4jCabu+rDz;hQFllMa%P0seIa1aK9X5?|5Ra`N(XG}$x5`~_{y1;3rn-(@s3 zWGRegKI6*KEzho2ma2IvSEcpt7Y#he#?qoEU(PF}7Qd{F_3W4xT$#BNMzhAGT)WzqKqT^`=H ziR4ZPoX0SsRh5#F8mg84TAa&MRyLzncs$usSI>@>6m${<%G~{l$9>!J&8i@Y`@`!4 zF*#hX!CjFmUs?z)Y=L_2O>}EgfVO%|{;-U80Mn}(ei^3Bo9I3LzMsBi z&f4MDx9W|Fe8G%$5BVrg2!tRMoZFvoRi|~#YNPO`;7=P#h8UF@zl;W3+|FfA-7|kJ znr)--q!&Qi9RdIoWtk+2Wbw6rSyg&&7m^Iq{B3G=nix4@%eGZHzJ>b}8AT^@L`@|B zjYvSFtYD(-G!}|!{qd^Uyi-JB(cBig7YSC$Pk^~!)MR45Fg!;;WWErH;IXmEc^8l~ z;XS^w_bZjgC0^P9WDupjAvE2Lc+H8NT%ap())-~#CFNwOx16f`eLiu72I4JC$Sufi zYMEr5*tWGc$A<=YKS$@}nx3i9-vyv-pEavS!7TmTSQvr2ssfq!I$~*Qg>b3_KH2o*$)fgoUxm$q$Q))pRSej z`|zhRL!m=jVFW&3DDsU}z*Sb->jb6{JI!PH`b@Y&kBpQ*H!)_jyMs99LSQJk1G(-& z#rbg*wkj^c%80sl zh~~{A2%#U28q``0`MuGgMT9LoYVcT zyvK5_PN%B*^K^8pKeb33#qaz^pmYbKeXXjhD*0G%+nkbm#7u;>aN5EN`a{5#TAR{Z zR$prBQ={9)O4!dmvF{zF4AFY`YV`Cjga%`OPaH><8h~n#<~W0_7;u)VZe~5-$D)qq zQEInaG$>b>dJtgrgcu#}j8~1bd%~zw($c7ia$a+sO3^wz-xH{de7ATPogtYZQuwj9 z$w@n7mtMob=t)&2!C;_jsijf!=c5<86Ro3QFmTI(s?g(u3%QY#%drhZOC5)~6Cb)- zq$`YtRM#LJyydK{thawl+ON{RN~f(ZyI+es)C29t)Zj9Zjlx>+qLePeL?=jLMG7f! zE_lTHiJJTgIE=FpRpPfR#tr&^pNJMm3aY}YonPJk+&Wy#&_m_7I_kIza~uHG7x$`F zO>SLRw6D&i+q^tR7qb#Z*wc-%#)Bu~=PB4dqr0}onpQP;k}$3*Gf}yIjpwkf1BIcd zCm?TVi{j4|pq4L%@}ll<|HRUK9D^kLN)=kXLnvxGQAsGw`p<&nH9BCk0^x~KG&t#1 zVOl+<>R(gG%YMf%B&H7>Ww`$OBzIZ<{Ts*R((kcj(}#aqS(!oyQw1D!3LFZk2hiCp2@%@-%LcYD->#G1MREESC?>|XOnQkKg!xq=q| zR7c3L3mWrubDjF$hX1Md{kc+<`rZ6u>F=Bh(Y#p=lU;J9-)FE!mN8Ct!WWJx2&k~x zBwMI@UtlXIIo9*!w{$}h=2fvWqi(b1X0OmJTsz2IMAs04~{Q3cAOv2_iQyPSg`eY)xnEsnJBu zHUS%%DX6pg>t3u;s>_^M?tPHej=<`<9BgRE1}U==ad2yu5*jibFhO6+P? zi1^PX%8Y->Us7F7>#W=gT39`Jyc&E9&$UOXAaZu#sq}Vj(x-!a({S{?r)g5 zEb5nXq3S%hHjS&}1hGJzhA3Q*hH!ClH^F$`G~QzT`3a>iRKhQp(yoUZlEM`twF6en zf)4tWnSb=J5I&UvY4R_KJub^ggw?-$swkdzu+nO?r7@eICOz0*Aa@XDXDFw5Z)7|) zbb0AM)rzy&=!C9OLDjcrut&qCm^b)Vpq|8aUYD%ySG%AWb~7bilZ+quD>RvHZ{@#r zwQ8)0HKL=Z6I8L|Q|sB8^xq}<+k!3a$IfFcJjt$kqB#luF2DE@XqHV&BSqFwcGstD z7b{IqcY@lEjzIYAf~xiFjQzK3LVB7o)OnAQ<&P!JBhq30JB|Vl0`U8vkNLv2i*=RD z>5dF-a=L=B>YJB_0vo~ zF;QMuY?R?7*|&AbXEhU{Jt``@e`?L~E(ECys^xvL&FUL<%eO1{`TzZQb!8>0^q4IC z?Csq0K0lwUBr_5FEG9RpiJ`WgHKn9(t9k|tJc^qjkIrkm6wG5i!zuvFQSPu8IWIA* zRBMV3V+O&DCQ><$US(#;op1jJ+rBJF&=T-w8*$ip%nkjsm6>t6^NcrnW}1knL0{O7 zf3so!E?{Inu z;&_I3H?IyiQGKSwL#%l9A%?8M@W_g5z$?E-fd&s3Wtje{gMErh+?LThr!a|FA$0YO z`VlKqa6~~xl}ScH8r-Mgwqy6>loxPA&l`2uK{aXk0g6tFl&I4JUn)OoF;m)|W$+GX zVE8tqtVd;P;zmgB7g;>2q7jyl z((1jYy`QGYu}{mDt-ZdZH&~)d3tFX?rYV2$dW?WCWaK2m)l{fxTwSQi8n^tN)esDK z{>?mHb!V8GuUhwZ<@Gb*XW$V@=Oj&Ct$nu3+e7sCj=G7lDfFBBL_qyjb@5JqmIL@g z`j-?GzdcadZ}P+>O4PdRZFQ_OcMo3Y&U`R5k8iCB&N;2O8z%rWnlWm8YUCg=iiqk* z1zGEAORYX`ggA(Mr-<=wZ*r}%$G?-&LjB)A5Xq}*=Dv~HYR&MD)0S%LOY~WLtym7- zm62D}m+m7@`lKFd$RK=|c0Kd{`P)8X5CPJ>czkc(4+sP8ddR&2n$DbxpcZObahdCW=7#e zQ%u45V7`ZgQFr^CGgRUPaW8A!{hx9L#Y>RqhFb^BII(+-J1wVNi*|k*_y!6vDAzT! zj*2ZGKZa&Ns)kOR3m2U=VihQ2 z=O587@A5N`NWy=ur`jNJNZangT&1DUD-1_j{uhz!>$H~vVtXE*AFS;M%C{$YvkMLh zcx|`${t~|5|A~__fN$6yCc@959H`KU#Ct4b1L7%lFhjGG)YP#2L=N;Zb=9x)xYWaJ zxlh4Rrt+&cx4uaIvtM7Y-n~MLmv-TTq!<}3S`W-n0j0iPZ+S}&Cx{k;f&^-rC=83j z|AzmuIH8T@I4D(@Ew)JaxFxX-?f(QG{28%SDyFrdXGRVdFZ}9n$0mI{yDM62-Z{@h zK90kNcLJ9v0kui&?AW#A zn&n|u4G?LXy@FD_J&i61m{6y7UtKlK(FKGDR~>L39Tre|%jGZvk3&js)E0VCO$a|P zINgPz&EvNAPOz8v#lK^6>FZcqgO+FsmWmY{RRsguz~eK>JE5@RB+9a8^U6u-TBnjy z00ll^{GNjLxfG+SfhsRn9`G6c2I#2n!&=iDY%Lui=@A(_PFQ$;Pt#W#7Hi!$T-FIn|O8-I_&+oplU2x+HsuGMYa{QAq@!187 zhN*mER{cnsZS(hYAvP-hxbF?27cCm6bpWJ%W*m4`N}x=8HK4BfiC6^brXPWw=>dp! z+{PG(J^CKqHh;-BahMl$s9&a=-9^HJ{5qi0M6(jWlgMRSCxzKxFS^|#OkJFB9EPIK z|I8RWn>fo@i!|u^!bMtF*gWs@ITCV+8mzMM@xJ$HbPt@PH>IsXyn{_wgA=D#P(9BDn@J+ZQ*4&$DY!3>FuyY(_Lg(``{8>O&+(HA zg7nMmsu>P2dwuyH!AI*C+e>h*O8Ph>L6gAu6nXO&-EBu*NFU)HOeLMY*cFE;2k?xX ze-RI=Shlxl1r(Kipb=I@@8xL*U=f%@jZhxXe(VzV*?t3U? z+H&*?FOH)kbf9QROY@2)R^WXK2lcwnd_^%N+9@(G8ok_^71QD8cEA_A9MROc4sxR~ ze!Me^U9I$JySP({$E9LIMV>Ml?-6_@2`2%|;a{Tn_Vu-2T(tX)1`Mb38GC!*+GWg& zSA`|t-k)-C^SggDjFVuStQ;iu-$+r*miv%de% z!M;-{Iy$EaPtaaT>OZRjL3wQv)aPP*jUV$#M%mC(1gIz|ZqL)E%h(?}Ztkh-V7Pu4 zy$@~;BP>zI!pFDhMqQz}e8~|#)|lbb?`4Rj_f7yDMv0qn_rg?SCXHo&N<}H+s5xx@xz;~{{hnHC1o)wApjARJSva_<{L}1Kwe<_%-6WQyIgb~K)?0a6l@>L$Y<2_Os;tSV^vnLf z2lN^zE^@MA8PP~xdP>~Yd7Zh8=KXy2Umf%jjTG)H!&z;=Ye?$DXYc3Jk1jk!9(w%3 zuC4^U*IJLd(|D}eMt4uW#;RCtFxAXLzPGjE{|uA1!G)01MM0+a6#?1LseRlCMfjiS zUID@{&uv#*z4$~=;{5dY6xR4d0fewqXJt5fMf;L9_kQ{YKURD(y(AY7xEGAp{|i+x z*y`$Df`J6AQ!;J!aoCJ7u5owg4P6(VRy;$5xcG5-$kwXZx zcF&w8fT7@%Khg(0TVrHu1`J7K*jd%;_0P81H+tCb@BRxPfXfjr8pm*$RBRwJq9YG~ z$Hj#vN2!)@*pIR-6O(rSjTjZx1pU3&-mNX`?srMARz5&C{&gC{5NRjsI`Y0q#u9rbNLM znSA<5%FADqfZW$mi^=N|7)`{+)RzcCH?*B@z6`Br=;(#B2z`CM2H(3@0{CBDy)&=T zx<>DNjRLi&7PIQ6tI}vHM061wGfrxPBMMAUM}d{_1VxlUZ4;hO}>{4EL@P27~wOz+HN+Sw%^eK z(z}nCT`qHq+9b7}g$H3MG9q4t$qDu_6TY^7)5!tIqA9?FVY#J5VK3n+DB9{J>gZ(A z3~cSUeJnmq6n%i~zR%EBw=hX5G1P$@5{!Z!$q@r1-axGl{sjx-A7LIg-7C*YoPsNT z0bSUv{HU^vmEdVSx-ozR5AVQ1r`<~nOhbkI=5cDOT2=;89i)+mRu}ahoE~j_`%mw~ zo(lsaSQ_m>r6u+yC1nR+`>Ep4$WVF3to4&~PmzBTvsg5MpqjPfb?j14x;aJN$KApNNxpJM5@fuVbieeX1uZD(2_w-7S2s5;Bq{Z&xVewIwRRe=eVyf9 zBB#T((+L$Q&oFB z;!y2;qxVtu88~&{N#gq>bg0z!qVZ9qc#IGUiK;hL50`yXr&Zv<=s&ZY61y3Ed|Ir_ zv>Fhw<#sPw^fRkEvb5pCxh?u07|TprtxY67c}p6rIqt@sGDUjMiTBl%CIF>ZDbvur zcKp9`Spsmv8ymHC9nQD&uoFX^Qu)VVMUB}UpPtEI`pcdy0G~g%vQeEg3pEg{>$A!P znK;Xfsm8ym%w;aBat@XoMNbjJbEUK|w*Jq@?0($w9jwVH>2_0BvL>k&{nf2B9H4hhrEWkl4-6!`oXnYqf|xSQb(( zQ~wwFaJMe6PXJ&DA9v;A;@}dHCNl!WV08A+7qp0^6B3zBggy_H8+=cC<}MeQu6`nG z_>@`EXpuWMzpfs(hiO(0>QJ~pR`1)=wY72ReOUxVmT8nxWKkU1wJDE*JP^tR669Y2 z^?D-<3yh$kS7>daq(Lx5OlqpYd&S{wT|M#EaT;qx#QPT8@|Mc;wJo$~y}bZ^l0OCj zbzCP5lrVX3tUd$*ZTBBMF?qQf5eX9r3iW#WGvRjP#@eAPaZvnuQtdMfaDr-~8nfbe zrx}6a@%1=k*YdWkF3EzxtCL%s8t?~DCBm>|K$Djk9maH-DJi;}r^agV*#h`xPe>`q z#PQKAc+*odGJ4MbvI59!p2T0BPxP;=#d3OYZ~ymwj1WD^ZP%t*80iTUZDzQMKMV_m z0+u35N#^@N97cvckm`fC=pY&zvs@`9?_XTG#&r~E{zW3E9w}MreVw5-k3$} z&um5Cx6F&b?(XguY2lkAx;mj@(iG&sY!Xtn1>0cc=m)NpS%A102QP17Z?6nM*!X`p zxdX{5ENml!FlG>c?c@K>Y?(f z=D*V^*B}K<%fV>IQ6}wfLV2j+pc+*S@Ah>Mz2Zc2{As*Lx=yVs>1yRP$G9+v4ixs% zAP`>ys8F8$nk2GGbOp4fL&w@WH(Hy11j0re-le(b8kGU!dmG zk*&kiGDw@c#4EPtJktVa#K$j9r`mD~F~h!g zBIR5Y@_#?~SLe}gFp4n50GRcQ&s9-zlsVnzV1cF3^(i#W7goPjo2Nn!x~xk z4Jr`Ou)x5m6a?eeDRF`NlnCvYVlIeWBuO{hFuF<^ZId>0;)UquJUh;!PiP-d*x3!~ z-$lJ_(=^TXpbvZ%$!h2MAzRNT=$qCiD4z!YW6#nrFpRZP?7XexQ;(>bjSWttcUx%! zR(SEJml52R@;oJ8efDvTAR2~o*sNpYQl@6>M$fVEk*L7xdK?%|MqP0u!SSVbp;a@J zS?FP^1qCuuCG#r3=_!Ny>)e6%+jhTDX*`GTm5*B*hSAm41(XJPdYFR~)!@~oE|zbw z58igE)ohAG6E-{w&R0Gvi4Y^DgP($c8|K^(Ln*FgY_ssO(kd-xErmS9$kE>GuPL|$ z>KYY#84G*-ADox;?6V}Hq@)zVMJ|)4EtLm#vX!-+;BjuSOY1w0i7_x);ij5UR#ux7J|`4f3$%em4e9_eto|p#STbiMx*Fi;; z+n6+#en1&)*-a&QNod?nN~c0vzgoN*-1`ReCZS09)B^+pb4POykC3EqUDV%KCH_mJS0F{XWhq+J7*f1JbI{$+<-G`>;XZhiOVsw>9g*B=_&e9k9jCvkM9(|+i z9GUex!i9e=2AW-z!z8Y>W4`;0HpC*3r)9kMA684Y$DO- z;)`bxE6mJPze{dHE31iN@(}G9aw(WvIi>oX)>`3;!Buaw6r3~BA+b97vA-U3B8PB( zsPoLHV718$^W(ZvXxL@9?)AXK{gF?7lnaGEkXdmqeC*#1Dx9&>V>0zG_hcwc!t>mb zS!g`lCneWsB8f!LyiQncQhHby+Z1?VG+pWt6&>^50)fPjJ1FmhK$IQz=m!t%2tbf( zBObzSJVjQDpexasn@6J~Eg;=A&cg+L0g>kWxJP=PSvYjI>cpmKp^m0gQ<0|3)f(RA z`gYv}>T<@)itT_;;b*we(s>X4`d~#~!Ec=U89t#27w-b&dFeR_tq)M-xPEMQ1IK|n zuddf4)?2q&#HWLd+!gvB$G>hXLGn!vOqTP9&Q4!# z*2*};axp+*;V=7)Hp(`A0fLNgC~5m?i+Kk!C~ZRT`+FI^{m)}pUW@E3C6=n)aYZd6 z=9vzjBFTfRyG6RJJD+!6nj0<7*aGHa$DEP0Ku~By#x0o3M6)NP>&O8n?{i4WZwC5) zkDjv+1EZ{OmIf!U$%xV%ELYL(c_RS&njp$(G(Lojn2K?tx=s15wFp6OU@G0)0&&3L zAdSZqaV`O>*Cgbg*lNO{wY1SwRbX(iv6-_81uex_D*bnctyTJW4dJ-b-PvU;UjJym zOoh&|IoPHkef;{$9!3>Nv>8{E!@B2MJdAN6Ua9(A5v$aWGs8E9b25Ci3=ae7!Lrg(?1S?rZHAg z)@gQFfKa&d(e`rYj5cjdG(9;AD!>T(j#ESyhWAPE7=mP;QK2AeYBUe|FnwlDd zK9vQ(Sv|5>%^?XmX=^sxJ&1L}Uz6M8RqOiqE0Ovtlw@T5cDtYKxhoyIo^kx6LS^`< zwDmm4>W${B#edvYIWR;3W5nc*fyiq5(6-ZLk1D1FKKyL^GsPofk<)=9Abbo^;(`#E zamu+ZtM0^R#-(ijj5tQoM{oCCOxL5^=Rd4wf^0c?xG$u?9!sd3Fan9iu zbb(8MJytTTXW#(>F@fh>*qK2;RRMXjCp;S7%KiX*P!T8|3p8#CK;=xu#KSg84T*Sx zX;#k>>UF9J_|~h>k7dGh0qfpAPCEXZ^A(U8HdfZ5)vKQmSMJWI!+ig@ zJaE=yuQIUxsQpfpW~Ep@#dP-8hLM8Hnuc=dPo*1X-5I1s=f_A@06wr<_TjDD@4NnUDEL`I<5;^`~ zrux{@1eU`Gpe*1mT^;QR8nwHf&~#JoMpPa_EcEx%j*CrDjz0 z29R2sH_wbLo`+JTlOyJu;dA`a#U!uR7Vq^Y77pGIQl&*qORn80Z`}{wJbiqE1ipVN znTm<4bZ-(<^;1;R_uVBm0T{tA4Jw19Wt>LKeZG`V_m1wEnd3ce1o167o;rvf>Fc>Y zt)&)+|2+|Q!39$>LNa+eA)uQ4o}BryS(8ng5~h#2DSu!k%!KP%evaev`pIA!h+dVj zQzB3A_j|;GiGTKAYDvTIr~(cRq;nK?-}R8v?Gf-QG&iiU|6K59E56LsJolmYyU$g`!Is{Zx?QbLf zwElb4J^uIH4mczMpr=@YA;uOHWmX7MigD0tn%@)fmj! z>S|>!>TI*8>3sGRaSn>3cfo^z^O^m0PY04)1_aEFab;EB*LhM#n+4_acY5c^X)e{L z9}4qJRbg;~J<@y-=W{Nz%jJDCvYKZi-@R6zd}Sfxthbsb+4fk}wpppk`PJp<;TdD! z>hof`(U^MKch|1Tr%z9igGctXjxnWdimw;al`rCb&-L&hq$~=E1f$)Y9|TNU>g~md zfiye;H?49;6{yCA(!Mku+l9?Lqv z46w1rfx1+SN*cg&85GB+3XI_eXKmF;(y=}(CIyiTXCdO!P z4#fLUCB~SGG}LVRPz3$^3k~XZAQf%AJk5d@2uBtPAMjm{1w7dudOsQ^t+GT0`8gy> zSpY)K5yBf8Pi#0S4DtN<(`b5Wsk38h3uooD+zsCE^*Hlsvm<9dZeU=3l8!LB@WuX$ zqR?x1s~ic9YNK(JK919YY7KFyG{ojM_TduE%-sC{uiOm!`aT6FAT3#LL@Ujm?CY7<>gzkEfWj##E{;v@6mD>A!a<0{hPxRk> z#RkIfUwMu1*`&U3@yif_4Vx1uFK&(Ie=iYM4v?=GXP?`Y=K!V=Yo|@u{QtOIZv)hrF*HM6! zIR!s=h?wtrR>hgA!+$aM<-_fLd*AP4{FUFE_Vxam31yMPgz3By-|y;DyPFcdg{e#FW=`Wf+?Qt0Bl&(-#<=sxwz4GI}G zbx(WSgGk@Di1+WTm2mHozO2^010Y(7bVYnGO_<1_8)ynu#Q=vJ$&`PUwJXNBNZmh| zG4wQBcSx_dS>)&E4+d5&KihU#QUJm>_~E3@&CuJ1$PW-{-W)qyGB)2Gj1A8%?&#*| zrAg0Sf~n^4*hq%a3ga&gU6o$g^!O9 z+pR}ib=I2eHCe6$NWz!ESAyn^cN8S+#fW*%NA?~9xVb>bg6gVqn^KO52?xCcTB2?= z>o;m^5;s(8)xtK~?l9|qSUO(U4xPFAj^F8N)?M&P%T3!iYuEZH&o|wvMYdE;n-|TQ zUYpSkx4qtt!i-;EN$GPzg9*k_ZCHy(6I8Nv{HsELCGKAWlNiswH{4AN)!&}^u}jy0 zSA%LCm-GQo`1aTN>FStNSn@prE328@pPLHKQiBGo(B$!dBRBnYeG>z9$Io$&mR zQ#PvpCGf=Q-jvtgx9y?=)s@O^qpN)#;^2q7oFijT*L&J#*CVfMNksL)+?+ykjX$g2 zbFRV8h_p z`$aE4TpXOX&l_Xi42069dEOF&d4$&A!d^ufcg%*_wpjGN@u5N)bFhiPPK*Gcf5<(r z%e9-8AjY@u(>j?4!^6p=-!s!!ZKO}+42wIoo}?$~IopwD8|S*ffHZMEFqDi2V}WQP@qA*)^fCb z)bY8*pWS!w&yVZPWbQ-JQYyPH{WosS2iHe-jrtr8?liC7%=ldUhGAf<5Q3xuI-p%w+H~~y8D8@B z>4-wXq{0lzB{^N&Y;6RD^d(?B?`}r)ZOr7z+M7v)mG6;(UC03`%vhG6qGsh;=*>}hoPCf1taPh880nF zc_`9VP)8{2w*?nO($tZJfZT$eX^}(25gkua3>c!a;(6cDeoA<|_eU@C;Yvv}*#(kw zct=aY@>8rGNe##v-v1r8@0vj@`kn%{7avoEh7LOfOHL4dSzo+4@ztMKYC{S1lTRV| z{x~%eGjQ#?KT%xu52HX`Ta=H7>*H&b`@?0VE!6SnWLCtNEDtu}Rh_V0i}>)~grZqR zpwyJtU;I8yh_2Tak5*+j6cvnu3O#?;t@ZA$DY3j)NCz@+_ip5A`{lJJyWQ{au}6Ln zvOpe6DGMlVe^53u=FrsZX!lubEwgVsuiqUV8MdvPvC6J9W&n}!@S8Vgj(GUai#b{& z#2HoPM=SIjt_y2M7w*Gtb$ETM=WT}Ccd5n3rhmFo$L)-!MHepnj%YVzaD~^jF#g6T%yD#T3M6al+ zVS9sn|LLiqo{S#(y#Qw3t{f*le0jJXeemubZdkfF0d5ArA}T=%2iYiE@y%A48rL;5 zWWC+^;BTY5zsUIz58DFfS@%cjGl+X##QgUG?Vov6Hx?~_L>`>8+AHk+j?gbx@O*<` zi>y1O@PEcf-zhALXk0o7dQ!j%vB^kJXYa1sHeu&btS6KhgJCfkCAF=O{C*qB7%o1u zkV~B2zFD8n-g;X8X`ANw-yQWmK-?L`{|(rAfGuOS4u&vD*=_w6ka6TCY+pr+8_xmbqPX#lEO zW;ieU{v9Ik2OWI;lzb-$4WfREv&E){TwNKlwO42{<2;q9EY;OxgsOD=i!(_>ajJL~ zx?ig%LG==(^$vnto^pLVsU=`}g73BH=eG~c`Uy&I!;b|fV;nzu4MTbXPR`)5xgIE?z3X|rw)FxVoh5`Kemd*x;Cj! z8PVEe)yer3)qZX3Z#zoVY4ulB%#sB~%M^!;i&sDUTiW-l^lY5cmO=kL8d`5kFLldh z$V0Q!Z07acrIk0*YgIora-ZKy_xT^RF8n-I?GE75vTS|vcll<>Y))4R?V0$VIHrpH zPh{&5oIO3^Np|93F8p1s8Z*U`?|#y#oo@S~ere+Vf&R^mn?ZEQ4A#-@7i7h6FH^Jb znT@f`Wscsz0c)$8(XnmFA;A0`r!Os@`kONRn?TNGB>#!3d`n$PzfaG>QlWxJ?XY~# zC$jbA;TmkSQVr0W5y1X~XE1v|W}T!zj!id|rt+M8({N{)_pl3JnRjFBCMU=l6>zQe(S z!3wWJ!AR6>J&9#&-MkRWOK!Q8^gh)AQtE~SejI>A{nG&d{$}#U_}1C&%V-X&7bptV zw2V-Ccy%!(yf}XQYwbbox5e&=MBNelnAm7t&7s|+vxMu1gAD7a@9Ho69(DzGeReYx z+}vE7GU~l-8{WKmF-e-OL2h&_`7VkYC+!_Q=PN5qB8-GIl5QPtgunkQf0|@94wd(4 zMz5@-aRScb>uDsXSAIt?yP6$2d9m*b`?{Og*VIarv(!wswVSjbFx>pL%k&u$c-VcA zEl}%yf+sTZn?Px^D4j~G)4`x~XHza-M|Ge(@ASrVMX`I64Lt4G^5f!bc7hSasDz#n zJZn7xdV+ZJL+_QS@LR{>@=wj|i`ToFRaP!0dBV%#3~+{x*W`X^lxX|{?%Y*AeTVGN zaWb{+qQXOxhj@PGqtSFc`xqlvhi4W_)4|ad-V@p^xNEm^wIgId^5?KO2v12Ld!16q zZF9e@0xj78x%ZnCJgxek0>|xY$($*Y3u{ix;}^dwJBjHfKB z$F)B4qlJ2te@3T8hzj1OsW3d+w8}J$9gt~5;2rw&`rR)l*xsu(v@~-Z&K;mgrsSbI ze)dlA1h;|&mlus`0iUgR<|U&ubK_-)0>W`Tr5IZ;sa29wk(3J3;_>4O<0 zD&LUPqt(tF-vRPmeTD=C=Q-p&^m6Z*;;8hz183-* zMOm3^BfJ4r(E0Qj&%V;5b`&zlocZ7)aQ6>PobliH>SttfcLhe%HmFMQ0wCP&J5Ay1 zt#1Y-AnbR!5#tNfy`T9da`%I_XAuy?U!&cmW_i}H-_dqRHf6SFbLi>So+r+JjWRkY z;!)|cg4md@Zzn&{JbU&m$vtzGwt)V9Rn*UfJZk*I-75(W0oxEoYgJq&!7(-Fqz~z$ zge~DCq-A0V6>klW{C6r9DUnPhy({if}N$U0S-20N{oC@^FFnG zYq?BsFWD!(?k{7)eg)v;TL4;k!PKnGc)^u!a%n@YCpFU?qVeQ)c3pYlx^kUd>VR3u zaYRw)ko4;vZ~An;T|!#4g#3tIucu!*>O8HP@AnD!OlnGBjLg8T!q8}FbVn7oPNfvG ze0Lhpj{6xiBV$4)lOFs^#k)Q7Haz*Aw=9OvZ=?Jxqh_Yz5~2rA)`Z0Bb+SXxdM!9# za{uH=GR0;17>*P|2&_3>e<(yjeaGdD?gQbK*Op zp*OC82@2%p88<5F{yzbp1!4Ny)9C6V&O$U{H4&Nvc!3*NmTU(gLqoe$lLK(N8$+FS zu&7gjGX(sW*5skSzOHEN`8;N1dmCL|#94?MRue%app;@|Sw>39QihT9N=cBUDUJ$M zJqm``Tn06J;c6I#P9&U@ot?8^|MHij2~P(40t!N$yRE0rE*7H-9Dba@eHA!=Angk{IDU}685 zQr2D~S+bPUlE_F&u#`m1wqaWqR8@y0MA+n{q?+xD5kdsEZRdJ>x?k+?@B7_Dk34Q_ z$H!)3??3nBbDzBqvpU*$C*raF00LB#fu=ZAgt}P(;tY3LRS8J=V4(n>F>v55(vz=< z5z~#RVKoujXRKPi-tFn?elwL!o-|r}0x%MSWp$zok3kYrQms)+O5^mA`C1ap@(CVcT}2OFL&icKKrTbai53fB&J5O!{>tX@UfCbwx*1og9QD8S1dA zub{@5$qRXaA&4NAf^Aze;W)|T#|LFIy_jBXM8BCPLOBXwUwH%OW@qhL*x&a?dwa`K z+g1}&N}ouE=RVYqf8p0~7>jHyWqC6*>Gu~c=zZyuMe`rr^VXhy*IoN*96oXc_k3?d zqf0wy?tS24WM^fuZQG9Wg899_&vvBu3b024Q{u`*-4=k+8eoP}1j~}u<#5SiW;C=D zkGq*nhEx#uhUh8>l8%;FM6d_81XPo&b?x62eF+J;#Y1j>VmA3zshUf41qq zMwb?G7Me|dbJ276+B-lbFjHdA|eWf@S<+Fl0if)1Plpy@pyc&x4Y{%OP5};v0N?>_jPySrkihTba@eH zAZl1m$lBX)fwC3cq6^mxFgTRQvZa?>g;F`jh7|-6vw)P+B$A2p+Vyw2tG@abB$G3c z8y~@(^h|8L|G`F=ab8%z_I8{Y$|32*k#1?Rj}PXSA0Iq%V?IB0&BVk+));n3kTi7x z7Ls6yNL#k;wzi}Ox;p1>?e6M&Z1d*l|Gs|xHvz*~ee0c#E-m5=L=CHnS-)lt0_}zi zz9d5X6>T6WbZn3$K~ey0(AJtlp;*BWHe7gFeCzJK4Y^z%@pv59erARE<6pNgniwx$ zpUdS}6pMwvis!buo@)yTsg%@C%=TJRso}PC+rdnG`>QiEnHN?pTXt~Ap53fjW8K}~ zZ*)l!XCZ1>5f=n&Zv6&)t)ZmE@soMX@9b)=xNg7eRhIZVm=DHmBGDq1ELckAW466F z?!@1=EbHLd_;|5p#tZ~OfK)1pHS50J=yD>?Le#J#E-1cr$L&y3z;%6$jE&>wl{eD1 z9lIRQ4U!rrE=af}WhtFb7x%vVj&U3ZJzd?%<#Sl~ooKR}?))46L69cf{#tPW001R) zMObuXVRU6WV{&C-bY%cCFfuSLFf%POGgL4*IyE^uGB+zQFgh?W0{y9Y0000bbVXQn zWMOn=I&E)cX=Zr#I65^sIx;saFfckWFms?0!~g&Q07*qoM6N<$ Ef*2Y*XaE2J literal 0 HcmV?d00001 diff --git a/docs/tutorial/finetuned_model_to_module.md b/docs/tutorial/finetuned_model_to_module.md new file mode 100644 index 00000000..ddc9d8d8 --- /dev/null +++ b/docs/tutorial/finetuned_model_to_module.md @@ -0,0 +1,275 @@ +# Fine-tune保存的模型如何转化为一个PaddleHub Module + +## 模型基本信息 + +本示例以模型ERNIE Tiny在数据集ChnSentiCorp上完成情感分类Fine-tune任务后保存的模型转化为一个PaddleHub Module，Module的基本信息如下： +```yaml +name: ernie_tiny_finetuned +version: 1.0.0 +summary: ERNIE tiny which was fine-tuned on the chnsenticorp dataset. +author: anonymous +author_email: +type: nlp/semantic_model +``` + +**本示例代码可以参考[finetuned_model_to_module](../../demo/text_classification/finetuned_model_to_module/)** + +Module存在一个接口predict，用于接收带预测，并给出文本的情感倾向（正面/负面），支持python接口调用和命令行调用。 +```python +import paddlehub as hub + +ernie_tiny_finetuned = hub.Module(name="ernie_tiny_finetuned") +ernie_tiny_finetuned.predcit(data=[["这个宾馆比较陈旧了，特价的房间也很一般。总体来说一般"], ["交通方便；环境很好；服务态度很好 房间较小"], + ["19天硬盘就罢工了~~~算上运来的一周都没用上15天~~~可就是不能换了~~~唉~~~~你说这算什么事呀~~~"]]) +``` + + +## Module创建 + +### step 1. 创建必要的目录与文件 + +创建一个finetuned_model_to_module的目录，并在finetuned_model_to_module目录下分别创建__init__.py、module.py，其中 + +|文件名|用途| +|-|-| +|\_\_init\_\_.py|空文件| +|module.py|主模块，提供Module的实现代码| +|ckpt文件|利用PaddleHub Fine-tune得到的ckpt文件夹，其中必须包含best_model文件| + + +```cmd +➜ tree finetuned_model_to_module +finetuned_model_to_module/ +├── __init__.py +├── ckpt_chnsenticorp +│   ├── *** +│   ├── best_model +│   │   ├── *** +└── module.py +``` + +### step 2. 编写Module处理代码 + +module.py文件为Module的入口代码所在，我们需要在其中实现预测逻辑。 + +#### step 2_1. 引入必要的头文件 +```python +import os + +import numpy as np +from paddlehub.common.logger import logger +from paddlehub.module.module import moduleinfo, serving +import paddlehub as hub +``` + +#### step 2_2. 定义ERNIE_Tiny_Finetuned类 +module.py中需要有一个继承了hub.Module的类存在，该类负责实现预测逻辑，并使用moduleinfo填写基本信息。当使用hub.Module(name="ernie_tiny_finetuned")加载Module时，PaddleHub会自动创建ERNIE_Tiny_Finetuned的对象并返回。 +```python +@moduleinfo( + name="ernie_tiny_finetuned", + version="1.0.0", + summary="ERNIE tiny which was fine-tuned on the chnsenticorp dataset.", + author="anonymous", + author_email="", + type="nlp/semantic_model") +class ERNIETinyFinetuned(hub.Module): + ... +``` +#### step 2_3. 执行必要的初始化 +```python +def _initialize(self, + ckpt_dir="ckpt_chnsenticorp", + num_class=2, + max_seq_len=128, + use_gpu=False, + batch_size=1): + self.ckpt_dir = os.path.join(self.directory, ckpt_dir) + self.num_class = num_class + self.MAX_SEQ_LEN = max_seq_len + + self.params_path = os.path.join(self.ckpt_dir, 'best_model') + if not os.path.exists(self.params_path): + logger.error( + "%s doesn't contain the best_model file which saves the best parameters as fietuning." + ) + exit() + + # Load Paddlehub ERNIE Tiny pretrained model + self.module = hub.Module(name="ernie_tiny") + inputs, outputs, program = self.module.context( + trainable=True, max_seq_len=max_seq_len) + + self.vocab_path = self.module.get_vocab_path() + + # Download dataset and use accuracy as metrics + # Choose dataset: GLUE/XNLI/ChinesesGLUE/NLPCC-DBQA/LCQMC + # metric should be acc, f1 or matthews + metrics_choices = ["acc"] + + # For ernie_tiny, it use sub-word to tokenize chinese sentence + # If not ernie tiny, sp_model_path and word_dict_path should be set None + reader = hub.reader.ClassifyReader( + vocab_path=self.module.get_vocab_path(), + max_seq_len=max_seq_len, + sp_model_path=self.module.get_spm_path(), + word_dict_path=self.module.get_word_dict_path()) + + # Construct transfer learning network + # Use "pooled_output" for classification tasks on an entire sentence. + # Use "sequence_output" for token-level output. + pooled_output = outputs["pooled_output"] + + # Setup feed list for data feeder + # Must feed all the tensor of module need + feed_list = [ + inputs["input_ids"].name, + inputs["position_ids"].name, + inputs["segment_ids"].name, + inputs["input_mask"].name, + ] + + # Setup runing config for PaddleHub Finetune API + config = hub.RunConfig( + use_data_parallel=False, + use_cuda=use_gpu, + batch_size=batch_size, + checkpoint_dir=self.ckpt_dir, + strategy=hub.AdamWeightDecayStrategy()) + + # Define a classfication finetune task by PaddleHub's API + self.cls_task = hub.TextClassifierTask( + data_reader=reader, + feature=pooled_output, + feed_list=feed_list, + num_classes=self.num_class, + config=config, + metrics_choices=metrics_choices) +``` + +初始化过程即为Fine-tune时创建Task的过程。 + +**NOTE:** 执行类的初始化不能使用默认的__init__接口，而是应该重载实现_initialize接口。对象默认内置了directory属性，可以直接获取到Module所在路径 + +#### step 3_4. 完善预测逻辑 +```python +def predict(self, data, return_result=False, accelerate_mode=True): + """ + Get prediction results + """ + run_states = self.cls_task.predict( + data=data, + return_result=return_result, + accelerate_mode=accelerate_mode) + return run_states +``` + +#### step 3_5. 支持serving调用 + +如果希望Module可以支持PaddleHub Serving部署预测服务，则需要将预测接口predcit加上serving修饰(`@serving`)，接口负责解析传入数据并进行预测，将结果返回。 + +如果不需要提供PaddleHub Serving部署预测服务，则可以不需要加上serving修饰。 + +```python +@serving +def predict(self, data, return_result=False, accelerate_mode=True): + """ + Get prediction results + """ + run_states = self.cls_task.predict( + data=data, + return_result=return_result, + accelerate_mode=accelerate_mode) + return run_states +``` + +### 完整代码 + +* [module.py](../../demo/text_classification/finetuned_model_to_module/module.py) + +* [__init__.py](../../demo/text_classification/finetuned_model_to_module/__init__.py) + +**NOTE:** `__init__.py`是空文件 + +## 测试步骤 + +完成Module编写后，我们可以通过以下方式测试该Module + +### 调用方法1 + +将Module安装到本机中，再通过Hub.Module(name=...)加载 +```shell +hub install finetuned_model_to_module +``` + +安装成功会显示**Successfully installed ernie_tiny_finetuned** + +```python +import paddlehub as hub +import numpy as np + + +ernie_tiny = hub.Module(name="ernie_tiny_finetuned") + +# Data to be prdicted +data = [["这个宾馆比较陈旧了，特价的房间也很一般。总体来说一般"], ["交通方便；环境很好；服务态度很好 房间较小"], + ["19天硬盘就罢工了~~~算上运来的一周都没用上15天~~~可就是不能换了~~~唉~~~~你说这算什么事呀~~~"]] + +index = 0 +run_states = ernie_tiny.predict(data=data) +results = [run_state.run_results for run_state in run_states] +for batch_result in results: + # get predict index + batch_result = np.argmax(batch_result, axis=2)[0] + for result in batch_result: + print("%s\tpredict=%s" % (data[index][0], result)) + index += 1 +``` + +### 调用方法2 + +直接通过Hub.Module(directory=...)加载 +```python +import paddlehub as hub +import numpy as np + +ernie_tiny_finetuned = hub.Module(directory="finetuned_model_to_module/") + +# Data to be prdicted +data = [["这个宾馆比较陈旧了，特价的房间也很一般。总体来说一般"], ["交通方便；环境很好；服务态度很好 房间较小"], + ["19天硬盘就罢工了~~~算上运来的一周都没用上15天~~~可就是不能换了~~~唉~~~~你说这算什么事呀~~~"]] + +index = 0 +run_states = ernie_tiny.predict(data=data) +results = [run_state.run_results for run_state in run_states] +for batch_result in results: + # get predict index + batch_result = np.argmax(batch_result, axis=2)[0] + for result in batch_result: + print("%s\tpredict=%s" % (data[index][0], result)) + index += 1 +``` + +### 调用方法3 +将finetuned_model_to_module作为路径加到环境变量中，直接加载ERNIETinyFinetuned对象 +```shell +export PYTHONPATH=finetuned_model_to_module:$PYTHONPATH +``` + +```python +from finetuned_model_to_module.module import ERNIETinyFinetuned +import numpy as np + +# Data to be prdicted +data = [["这个宾馆比较陈旧了，特价的房间也很一般。总体来说一般"], ["交通方便；环境很好；服务态度很好 房间较小"], + ["19天硬盘就罢工了~~~算上运来的一周都没用上15天~~~可就是不能换了~~~唉~~~~你说这算什么事呀~~~"]] + +run_states = ERNIETinyFinetuned.predict(data=data) +index = 0 +results = [run_state.run_results for run_state in run_states] +for batch_result in results: + # get predict index + batch_result = np.argmax(batch_result, axis=2)[0] + for result in batch_result: + print("%s\tpredict=%s" % (data[index][0], result)) + index += 1 +``` diff --git a/docs/tutorial/tutorial_index.rst b/docs/tutorial/tutorial_index.rst index 9ebcbed0..44295ecc 100644 --- a/docs/tutorial/tutorial_index.rst +++ b/docs/tutorial/tutorial_index.rst @@ -11,10 +11,11 @@ 命令行工具 自定义数据 + Fine-tune模型转化为PaddleHub Module 自定义任务 服务化部署 文本Embedding服务 语义相似度计算 ULMFit优化策略 超参优化 - Hook机制 \ No newline at end of file + Hook机制 -- GitLab