From e2368644c52a2d0bb50930d71c0dcc8aea58d08e Mon Sep 17 00:00:00 2001 From: xixiaoyao Date: Wed, 23 Oct 2019 16:56:08 +0800 Subject: [PATCH] fix bugs --- build/lib/paddlepalm/__init__.py | 5 + build/lib/paddlepalm/backbone/__init__.py | 0 build/lib/paddlepalm/backbone/bert.py | 156 +++ build/lib/paddlepalm/backbone/bow.py | 63 ++ build/lib/paddlepalm/backbone/ernie.py | 170 +++ .../lib/paddlepalm/backbone/utils/__init__.py | 0 .../paddlepalm/backbone/utils/transformer.py | 341 ++++++ build/lib/paddlepalm/default_settings.py | 42 + build/lib/paddlepalm/interface.py | 173 +++ build/lib/paddlepalm/mtl_controller.py | 717 +++++++++++++ build/lib/paddlepalm/optimizer/__init__.py | 0 build/lib/paddlepalm/optimizer/adam.py | 108 ++ build/lib/paddlepalm/reader/__init__.py | 0 build/lib/paddlepalm/reader/cls4bert.py | 0 build/lib/paddlepalm/reader/match4ernie.py | 103 ++ build/lib/paddlepalm/reader/mlm.py | 103 ++ build/lib/paddlepalm/reader/mrc4bert.py | 656 ++++++++++++ build/lib/paddlepalm/reader/mrc4ernie.py | 119 +++ build/lib/paddlepalm/reader/utils/__init__.py | 0 .../paddlepalm/reader/utils/batching4bert.py | 184 ++++ .../paddlepalm/reader/utils/batching4ernie.py | 175 ++++ .../paddlepalm/reader/utils/mlm_batching.py | 175 ++++ .../paddlepalm/reader/utils/mrqa_helper.py | 84 ++ .../paddlepalm/reader/utils/reader4ernie.py | 989 ++++++++++++++++++ build/lib/paddlepalm/task_instance.py | 286 +++++ .../lib/paddlepalm/task_paradigm/__init__.py | 0 build/lib/paddlepalm/task_paradigm/cls.py | 60 ++ build/lib/paddlepalm/task_paradigm/match.py | 70 ++ build/lib/paddlepalm/task_paradigm/mlm.py | 111 ++ build/lib/paddlepalm/task_paradigm/mrc.py | 486 +++++++++ build/lib/paddlepalm/tokenizer/__init__.py | 0 .../paddlepalm/tokenizer/bert_tokenizer.py | 374 +++++++ .../paddlepalm/tokenizer/ernie_tokenizer.py | 417 ++++++++ build/lib/paddlepalm/utils/__init__.py | 0 build/lib/paddlepalm/utils/config_helper.py | 311 ++++++ build/lib/paddlepalm/utils/print_helper.py | 31 + build/lib/paddlepalm/utils/reader_helper.py | 226 ++++ build/lib/paddlepalm/utils/saver.py | 65 ++ .../paddlepalm/utils/textprocess_helper.py | 19 + dist/paddle_palm-1.2-py2.7.egg | Bin 0 -> 164003 bytes paddle_palm.egg-info/PKG-INFO | 105 ++ paddle_palm.egg-info/SOURCES.txt | 47 + paddle_palm.egg-info/dependency_links.txt | 1 + paddle_palm.egg-info/not-zip-safe | 1 + paddle_palm.egg-info/top_level.txt | 1 + paddlepalm/mtl_controller.py | 3 +- paddlepalm/utils/reader_helper.py | 5 +- run_demo1.sh | 4 +- run_demo2.sh | 4 +- 49 files changed, 6981 insertions(+), 9 deletions(-) create mode 100644 build/lib/paddlepalm/__init__.py create mode 100644 build/lib/paddlepalm/backbone/__init__.py create mode 100644 build/lib/paddlepalm/backbone/bert.py create mode 100644 build/lib/paddlepalm/backbone/bow.py create mode 100644 build/lib/paddlepalm/backbone/ernie.py create mode 100644 build/lib/paddlepalm/backbone/utils/__init__.py create mode 100644 build/lib/paddlepalm/backbone/utils/transformer.py create mode 100644 build/lib/paddlepalm/default_settings.py create mode 100644 build/lib/paddlepalm/interface.py create mode 100644 build/lib/paddlepalm/mtl_controller.py create mode 100644 build/lib/paddlepalm/optimizer/__init__.py create mode 100644 build/lib/paddlepalm/optimizer/adam.py create mode 100644 build/lib/paddlepalm/reader/__init__.py create mode 100644 build/lib/paddlepalm/reader/cls4bert.py create mode 100644 build/lib/paddlepalm/reader/match4ernie.py create mode 100644 build/lib/paddlepalm/reader/mlm.py create mode 100644 build/lib/paddlepalm/reader/mrc4bert.py create mode 100644 build/lib/paddlepalm/reader/mrc4ernie.py create mode 100644 build/lib/paddlepalm/reader/utils/__init__.py create mode 100644 build/lib/paddlepalm/reader/utils/batching4bert.py create mode 100644 build/lib/paddlepalm/reader/utils/batching4ernie.py create mode 100644 build/lib/paddlepalm/reader/utils/mlm_batching.py create mode 100644 build/lib/paddlepalm/reader/utils/mrqa_helper.py create mode 100644 build/lib/paddlepalm/reader/utils/reader4ernie.py create mode 100644 build/lib/paddlepalm/task_instance.py create mode 100644 build/lib/paddlepalm/task_paradigm/__init__.py create mode 100644 build/lib/paddlepalm/task_paradigm/cls.py create mode 100644 build/lib/paddlepalm/task_paradigm/match.py create mode 100644 build/lib/paddlepalm/task_paradigm/mlm.py create mode 100644 build/lib/paddlepalm/task_paradigm/mrc.py create mode 100644 build/lib/paddlepalm/tokenizer/__init__.py create mode 100644 build/lib/paddlepalm/tokenizer/bert_tokenizer.py create mode 100644 build/lib/paddlepalm/tokenizer/ernie_tokenizer.py create mode 100644 build/lib/paddlepalm/utils/__init__.py create mode 100644 build/lib/paddlepalm/utils/config_helper.py create mode 100644 build/lib/paddlepalm/utils/print_helper.py create mode 100644 build/lib/paddlepalm/utils/reader_helper.py create mode 100644 build/lib/paddlepalm/utils/saver.py create mode 100644 build/lib/paddlepalm/utils/textprocess_helper.py create mode 100644 dist/paddle_palm-1.2-py2.7.egg create mode 100644 paddle_palm.egg-info/PKG-INFO create mode 100644 paddle_palm.egg-info/SOURCES.txt create mode 100644 paddle_palm.egg-info/dependency_links.txt create mode 100644 paddle_palm.egg-info/not-zip-safe create mode 100644 paddle_palm.egg-info/top_level.txt diff --git a/build/lib/paddlepalm/__init__.py b/build/lib/paddlepalm/__init__.py new file mode 100644 index 0000000..c7b42da --- /dev/null +++ b/build/lib/paddlepalm/__init__.py @@ -0,0 +1,5 @@ + +import sys +from paddlepalm.mtl_controller import Controller +sys.path.append('paddlepalm') + diff --git a/build/lib/paddlepalm/backbone/__init__.py b/build/lib/paddlepalm/backbone/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/paddlepalm/backbone/bert.py b/build/lib/paddlepalm/backbone/bert.py new file mode 100644 index 0000000..05d0af5 --- /dev/null +++ b/build/lib/paddlepalm/backbone/bert.py @@ -0,0 +1,156 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""v1.1 +BERT model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from paddle import fluid +from paddle.fluid import layers + +from paddlepalm.backbone.utils.transformer import pre_process_layer, encoder +from paddlepalm.interface import backbone + + +class Model(backbone): + + def __init__(self, + config, + phase): + + # self._is_training = phase == 'train' # backbone一般不用关心运行阶段，因为outputs在任何阶段基本不会变 + self._emb_size = config["hidden_size"] + self._n_layer = config["num_hidden_layers"] + self._n_head = config["num_attention_heads"] + self._voc_size = config["vocab_size"] + self._max_position_seq_len = config["max_position_embeddings"] + self._sent_types = config["type_vocab_size"] + self._hidden_act = config["hidden_act"] + self._prepostprocess_dropout = config["hidden_dropout_prob"] + self._attention_dropout = config["attention_probs_dropout_prob"] + + self.model_name = model_name + + self._word_emb_name = self.model_name + "word_embedding" + self._pos_emb_name = self.model_name + "pos_embedding" + self._sent_emb_name = self.model_name + "sent_embedding" + + # Initialize all weigths by truncated normal initializer, and all biases + # will be initialized by constant zero by default. + self._param_initializer = fluid.initializer.TruncatedNormal( + scale=config["initializer_range"]) + + @property + def inputs_attr(self): + return {"token_ids": [-1, self._max_position_seq_len, 1], 'int64'], + "position_ids": [-1, self._max_position_seq_len, 1], 'int64'], + "segment_ids": [-1, self._max_position_seq_len, 1], 'int64'], + "input_mask": [-1, self._max_position_seq_len, 1], 'float32']} + + @property + def outputs_attr(self): + return {"word_emb": [-1, self._max_position_seq_len, self._emb_size], + "sentence_emb": [-1, self._emb_size], + "sentence_pair_emb": [-1, self._emb_size]} + + def build(self, inputs): + src_ids = inputs['token_ids'] + pos_ids = inputs['position_ids'] + sent_ids = inputs['segment_ids'] + input_mask = inputs['input_mask'] + # padding id in vocabulary must be set to 0 + emb_out = layers.embedding( + input=src_ids, + size=[self._voc_size, self._emb_size], + dtype="float32", + param_attr=fluid.ParamAttr( + name=self._word_emb_name, initializer=self._param_initializer), + is_sparse=False) + + self.emb_out = emb_out + + position_emb_out = layers.embedding( + input=pos_ids, + size=[self._max_position_seq_len, self._emb_size], + dtype="float32", + param_attr=fluid.ParamAttr( + name=self._pos_emb_name, initializer=self._param_initializer)) + + self.position_emb_out = position_emb_out + + sent_emb_out = layers.embedding( + sent_ids, + size=[self._sent_types, self._emb_size], + dtype="float32" + param_attr=fluid.ParamAttr( + name=self._sent_emb_name, initializer=self._param_initializer)) + + self.sent_emb_out = sent_emb_out + + emb_out = emb_out + position_emb_out + sent_emb_out + + emb_out = pre_process_layer( + emb_out, 'nd', self._prepostprocess_dropout, name='pre_encoder') + + self_attn_mask = layers.matmul( + x = input_mask, y = input_mask, transpose_y = True) + + self_attn_mask = layers.scale( + x = self_attn_mask, scale = 10000.0, bias = -1.0, bias_after_scale = False) + + n_head_self_attn_mask = layers.stack( + x=[self_attn_mask] * self._n_head, axis=1) + + n_head_self_attn_mask.stop_gradient = True + + enc_out = encoder( + enc_input = emb_out, + attn_bias = n_head_self_attn_mask, + n_layer = self._n_layer, + n_head = self._n_head, + d_key = self._emb_size // self._n_head, + d_value = self._emb_size // self._n_head, + d_model = self._emb_size, + d_inner_hid = self._emb_size * 4, + prepostprocess_dropout = self._prepostprocess_dropout, + attention_dropout = self._attention_dropout, + relu_dropout = 0, + hidden_act = self._hidden_act, + preprocess_cmd = "", + postprocess_cmd = "dan", + param_initializer = self._param_initializer, + name = self.model_name + 'encoder') + + next_sent_feat = layers.slice( + input = enc_out, axes = [1], starts = [0], ends = [1]) + next_sent_feat = layers.fc( + input = next_sent_feat, + size = self._emb_size, + act = "tanh", + param_attr = fluid.ParamAttr( + name = self.model_name + "pooled_fc.w_0", + initializer = self._param_initializer), + bias_attr = "pooled_fc.b_0") + + return {'word_emb': enc_out, + 'sentence_emb': next_sent_feat, + 'sentence_pair_emb': next_sent_feat} + + def postprocess(self, rt_outputs): + pass + + diff --git a/build/lib/paddlepalm/backbone/bow.py b/build/lib/paddlepalm/backbone/bow.py new file mode 100644 index 0000000..9268957 --- /dev/null +++ b/build/lib/paddlepalm/backbone/bow.py @@ -0,0 +1,63 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from paddle import fluid +from paddle.fluid import layers + +class Model(backbone): + + def __init__(self, config, phase): + + # self._is_training = phase == 'train' # backbone一般不用关心运行阶段，因为outputs在任何阶段基本不会变 + + self._emb_size = config["emb_size"] + self._voc_size = config["vocab_size"] + + @property + def inputs_attr(self): + return {"token_ids": [-1, self._max_position_seq_len, 1], 'int64']} + + @property + def outputs_attr(self): + return {"word_emb": [-1, self._max_position_seq_len, self._emb_size], + "sentence_emb": [-1, self._emb_size*2]} + + def build(self, inputs): + + tok_ids = inputs['token_ids'] + + emb_out = layers.embedding( + input=tok_ids, + size=[self._voc_size, self._emb_size], + dtype='float32', + param_attr=fluid.ParamAttr( + name='word_emb', + initializer=fluid.initializer.TruncatedNormal(scale=0.1)), + is_sparse=False) + + sent_emb1 = layers.reduce_mean(emb_out, axis=1) + sent_emb2 = layers.reduce_max(emb_out, axis=1) + sent_emb = layers.concat([sent_emb1, sent_emb2], axis=1) + return {'word_emb': emb_out, + 'sentence_emb': sent_emb} + + def postprocess(self, rt_outputs): + pass + + diff --git a/build/lib/paddlepalm/backbone/ernie.py b/build/lib/paddlepalm/backbone/ernie.py new file mode 100644 index 0000000..7a11769 --- /dev/null +++ b/build/lib/paddlepalm/backbone/ernie.py @@ -0,0 +1,170 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Ernie model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import absolute_import + +from paddle import fluid +from paddle.fluid import layers + +from paddlepalm.backbone.utils.transformer import pre_process_layer, encoder +from paddlepalm.interface import backbone + + +class Model(backbone): + + def __init__(self, + config, + phase): + + # self._is_training = phase == 'train' # backbone一般不用关心运行阶段，因为outputs在任何阶段基本不会变 + + self._emb_size = config['hidden_size'] + self._n_layer = config['num_hidden_layers'] + self._n_head = config['num_attention_heads'] + self._voc_size = config['vocab_size'] + self._max_position_seq_len = config['max_position_embeddings'] + if config['sent_type_vocab_size']: + self._sent_types = config['sent_type_vocab_size'] + else: + self._sent_types = config['type_vocab_size'] + + self._task_types = config['task_type_vocab_size'] + + self._hidden_act = config['hidden_act'] + self._prepostprocess_dropout = config['hidden_dropout_prob'] + self._attention_dropout = config['attention_probs_dropout_prob'] + + self._word_emb_name = "word_embedding" + self._pos_emb_name = "pos_embedding" + self._sent_emb_name = "sent_embedding" + self._task_emb_name = "task_embedding" + self._emb_dtype = "float32" + + self._param_initializer = fluid.initializer.TruncatedNormal( + scale=config['initializer_range']) + + @property + def inputs_attr(self): + return {"token_ids": [[-1, -1, 1], 'int64'], + "position_ids": [[-1, -1, 1], 'int64'], + "segment_ids": [[-1, -1, 1], 'int64'], + "input_mask": [[-1, -1, 1], 'float32'], + "task_ids": [[-1,-1, 1], 'int64']} + + @property + def outputs_attr(self): + return {"word_embedding": [[-1, -1, self._emb_size], 'float32'], + "encoder_outputs": [[-1, -1, self._emb_size], 'float32'], + "sentence_embedding": [[-1, self._emb_size], 'float32'], + "sentence_pair_embedding": [[-1, self._emb_size], 'float32']} + + def build(self, inputs): + + src_ids = inputs['token_ids'] + pos_ids = inputs['position_ids'] + sent_ids = inputs['segment_ids'] + input_mask = inputs['input_mask'] + task_ids = inputs['task_ids'] + + # padding id in vocabulary must be set to 0 + emb_out = fluid.layers.embedding( + input=src_ids, + size=[self._voc_size, self._emb_size], + dtype=self._emb_dtype, + param_attr=fluid.ParamAttr( + name=self._word_emb_name, initializer=self._param_initializer), + is_sparse=False) + + position_emb_out = fluid.layers.embedding( + input=pos_ids, + size=[self._max_position_seq_len, self._emb_size], + dtype=self._emb_dtype, + param_attr=fluid.ParamAttr( + name=self._pos_emb_name, initializer=self._param_initializer)) + + sent_emb_out = fluid.layers.embedding( + sent_ids, + size=[self._sent_types, self._emb_size], + dtype=self._emb_dtype, + param_attr=fluid.ParamAttr( + name=self._sent_emb_name, initializer=self._param_initializer)) + + emb_out = emb_out + position_emb_out + emb_out = emb_out + sent_emb_out + + task_emb_out = fluid.layers.embedding( + task_ids, + size=[self._task_types, self._emb_size], + dtype=self._emb_dtype, + param_attr=fluid.ParamAttr( + name=self._task_emb_name, + initializer=self._param_initializer)) + + emb_out = emb_out + task_emb_out + + emb_out = pre_process_layer( + emb_out, 'nd', self._prepostprocess_dropout, name='pre_encoder') + + self_attn_mask = fluid.layers.matmul( + x=input_mask, y=input_mask, transpose_y=True) + + self_attn_mask = fluid.layers.scale( + x=self_attn_mask, scale=10000.0, bias=-1.0, bias_after_scale=False) + n_head_self_attn_mask = fluid.layers.stack( + x=[self_attn_mask] * self._n_head, axis=1) + n_head_self_attn_mask.stop_gradient = True + + enc_out = encoder( + enc_input=emb_out, + attn_bias=n_head_self_attn_mask, + n_layer=self._n_layer, + n_head=self._n_head, + d_key=self._emb_size // self._n_head, + d_value=self._emb_size // self._n_head, + d_model=self._emb_size, + d_inner_hid=self._emb_size * 4, + prepostprocess_dropout=self._prepostprocess_dropout, + attention_dropout=self._attention_dropout, + relu_dropout=0, + hidden_act=self._hidden_act, + preprocess_cmd="", + postprocess_cmd="dan", + param_initializer=self._param_initializer, + name='encoder') + + + next_sent_feat = fluid.layers.slice( + input=enc_out, axes=[1], starts=[0], ends=[1]) + next_sent_feat = fluid.layers.reshape(next_sent_feat, [-1, next_sent_feat.shape[-1]]) + next_sent_feat = fluid.layers.fc( + input=next_sent_feat, + size=self._emb_size, + act="tanh", + param_attr=fluid.ParamAttr( + name="pooled_fc.w_0", initializer=self._param_initializer), + bias_attr="pooled_fc.b_0") + + return {'word_embedding': emb_out, + 'encoder_outputs': enc_out, + 'sentence_embedding': next_sent_feat, + 'sentence_pair_embedding': next_sent_feat} + + def postprocess(self, rt_outputs): + pass diff --git a/build/lib/paddlepalm/backbone/utils/__init__.py b/build/lib/paddlepalm/backbone/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/paddlepalm/backbone/utils/transformer.py b/build/lib/paddlepalm/backbone/utils/transformer.py new file mode 100644 index 0000000..d5aa5c7 --- /dev/null +++ b/build/lib/paddlepalm/backbone/utils/transformer.py @@ -0,0 +1,341 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Transformer encoder.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from functools import partial + +import paddle.fluid as fluid +import paddle.fluid.layers as layers + +def multi_head_attention(queries, + keys, + values, + attn_bias, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0., + cache=None, + param_initializer=None, + name='multi_head_att'): + """ + Multi-Head Attention. Note that attn_bias is added to the logit before + computing softmax activiation to mask certain selected positions so that + they will not considered in attention weights. + """ + keys = queries if keys is None else keys + values = keys if values is None else values + + if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3): + raise ValueError( + "Inputs: quries, keys and values should all be 3-D tensors.") + + def __compute_qkv(queries, keys, values, n_head, d_key, d_value): + """ + Add linear projection to queries, keys, and values. + """ + q = layers.fc(input=queries, + size=d_key * n_head, + num_flatten_dims=2, + param_attr=fluid.ParamAttr( + name=name + '_query_fc.w_0', + initializer=param_initializer), + bias_attr=name + '_query_fc.b_0') + k = layers.fc(input=keys, + size=d_key * n_head, + num_flatten_dims=2, + param_attr=fluid.ParamAttr( + name=name + '_key_fc.w_0', + initializer=param_initializer), + bias_attr=name + '_key_fc.b_0') + v = layers.fc(input=values, + size=d_value * n_head, + num_flatten_dims=2, + param_attr=fluid.ParamAttr( + name=name + '_value_fc.w_0', + initializer=param_initializer), + bias_attr=name + '_value_fc.b_0') + return q, k, v + + def __split_heads(x, n_head): + """ + Reshape the last dimension of inpunt tensor x so that it becomes two + dimensions and then transpose. Specifically, input a tensor with shape + [bs, max_sequence_length, n_head * hidden_dim] then output a tensor + with shape [bs, n_head, max_sequence_length, hidden_dim]. + """ + hidden_size = x.shape[-1] + # The value 0 in shape attr means copying the corresponding dimension + # size of the input as the output dimension size. + reshaped = layers.reshape( + x=x, shape=[0, 0, n_head, hidden_size // n_head], inplace=True) + + # permuate the dimensions into: + # [batch_size, n_head, max_sequence_len, hidden_size_per_head] + return layers.transpose(x=reshaped, perm=[0, 2, 1, 3]) + + def __combine_heads(x): + """ + Transpose and then reshape the last two dimensions of inpunt tensor x + so that it becomes one dimension, which is reverse to __split_heads. + """ + if len(x.shape) == 3: return x + if len(x.shape) != 4: + raise ValueError("Input(x) should be a 4-D Tensor.") + + trans_x = layers.transpose(x, perm=[0, 2, 1, 3]) + # The value 0 in shape attr means copying the corresponding dimension + # size of the input as the output dimension size. + return layers.reshape( + x=trans_x, + shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], + inplace=True) + + def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate): + """ + Scaled Dot-Product Attention + """ + scaled_q = layers.scale(x=q, scale=d_key**-0.5) + product = layers.matmul(x=scaled_q, y=k, transpose_y=True) + if attn_bias: + product += attn_bias + weights = layers.softmax(product) + if dropout_rate: + weights = layers.dropout( + weights, + dropout_prob=dropout_rate, + dropout_implementation="upscale_in_train", + is_test=False) + out = layers.matmul(weights, v) + return out + + q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value) + + if cache is not None: # use cache and concat time steps + # Since the inplace reshape in __split_heads changes the shape of k and + # v, which is the cache input for next time step, reshape the cache + # input from the previous time step first. + k = cache["k"] = layers.concat( + [layers.reshape( + cache["k"], shape=[0, 0, d_model]), k], axis=1) + v = cache["v"] = layers.concat( + [layers.reshape( + cache["v"], shape=[0, 0, d_model]), v], axis=1) + + q = __split_heads(q, n_head) + k = __split_heads(k, n_head) + v = __split_heads(v, n_head) + + ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_key, + dropout_rate) + + out = __combine_heads(ctx_multiheads) + + # Project back to the model size. + proj_out = layers.fc(input=out, + size=d_model, + num_flatten_dims=2, + param_attr=fluid.ParamAttr( + name=name + '_output_fc.w_0', + initializer=param_initializer), + bias_attr=name + '_output_fc.b_0') + return proj_out + + +def positionwise_feed_forward(x, + d_inner_hid, + d_hid, + dropout_rate, + hidden_act, + param_initializer=None, + name='ffn'): + """ + Position-wise Feed-Forward Networks. + This module consists of two linear transformations with a ReLU activation + in between, which is applied to each position separately and identically. + """ + hidden = layers.fc(input=x, + size=d_inner_hid, + num_flatten_dims=2, + act=hidden_act, + param_attr=fluid.ParamAttr( + name=name + '_fc_0.w_0', + initializer=param_initializer), + bias_attr=name + '_fc_0.b_0') + if dropout_rate: + hidden = layers.dropout( + hidden, + dropout_prob=dropout_rate, + dropout_implementation="upscale_in_train", + is_test=False) + out = layers.fc(input=hidden, + size=d_hid, + num_flatten_dims=2, + param_attr=fluid.ParamAttr( + name=name + '_fc_1.w_0', initializer=param_initializer), + bias_attr=name + '_fc_1.b_0') + return out + + +def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0., + name=''): + """ + Add residual connection, layer normalization and droput to the out tensor + optionally according to the value of process_cmd. + This will be used before or after multi-head attention and position-wise + feed-forward networks. + """ + for cmd in process_cmd: + if cmd == "a": # add residual connection + out = out + prev_out if prev_out else out + elif cmd == "n": # add layer normalization + out_dtype = out.dtype + if out_dtype == fluid.core.VarDesc.VarType.FP16: + out = layers.cast(x=out, dtype="float32") + out = layers.layer_norm( + out, + begin_norm_axis=len(out.shape) - 1, + param_attr=fluid.ParamAttr( + name=name + '_layer_norm_scale', + initializer=fluid.initializer.Constant(1.)), + bias_attr=fluid.ParamAttr( + name=name + '_layer_norm_bias', + initializer=fluid.initializer.Constant(0.))) + if out_dtype == fluid.core.VarDesc.VarType.FP16: + out = layers.cast(x=out, dtype="float16") + elif cmd == "d": # add dropout + if dropout_rate: + out = layers.dropout( + out, + dropout_prob=dropout_rate, + dropout_implementation="upscale_in_train", + is_test=False) + return out + + +pre_process_layer = partial(pre_post_process_layer, None) +post_process_layer = pre_post_process_layer + + +def encoder_layer(enc_input, + attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + hidden_act, + preprocess_cmd="n", + postprocess_cmd="da", + param_initializer=None, + name=''): + """The encoder layers that can be stacked to form a deep encoder. + This module consits of a multi-head (self) attention followed by + position-wise feed-forward networks and both the two components companied + with the post_process_layer to add residual connection, layer normalization + and droput. + """ + attn_output = multi_head_attention( + pre_process_layer( + enc_input, + preprocess_cmd, + prepostprocess_dropout, + name=name + '_pre_att'), + None, + None, + attn_bias, + d_key, + d_value, + d_model, + n_head, + attention_dropout, + param_initializer=param_initializer, + name=name + '_multi_head_att') + attn_output = post_process_layer( + enc_input, + attn_output, + postprocess_cmd, + prepostprocess_dropout, + name=name + '_post_att') + ffd_output = positionwise_feed_forward( + pre_process_layer( + attn_output, + preprocess_cmd, + prepostprocess_dropout, + name=name + '_pre_ffn'), + d_inner_hid, + d_model, + relu_dropout, + hidden_act, + param_initializer=param_initializer, + name=name + '_ffn') + return post_process_layer( + attn_output, + ffd_output, + postprocess_cmd, + prepostprocess_dropout, + name=name + '_post_ffn') + + +def encoder(enc_input, + attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + hidden_act, + preprocess_cmd="n", + postprocess_cmd="da", + param_initializer=None, + name=''): + """ + The encoder is composed of a stack of identical layers returned by calling + encoder_layer. + """ + for i in range(n_layer): + enc_output = encoder_layer( + enc_input, + attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + hidden_act, + preprocess_cmd, + postprocess_cmd, + param_initializer=param_initializer, + name=name + '_layer_' + str(i)) + enc_input = enc_output + enc_output = pre_process_layer( + enc_output, preprocess_cmd, prepostprocess_dropout, name="post_encoder") + + return enc_output diff --git a/build/lib/paddlepalm/default_settings.py b/build/lib/paddlepalm/default_settings.py new file mode 100644 index 0000000..4f003ea --- /dev/null +++ b/build/lib/paddlepalm/default_settings.py @@ -0,0 +1,42 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BACKBONE_DIR='paddlepalm.backbone' +TASK_INSTANCE_DIR='paddlepalm.task_instance' +READER_DIR='paddlepalm.reader' +PARADIGM_DIR='paddlepalm.task_paradigm' +OPTIMIZER_DIR='paddlepalm.optimizer' +OPTIMIZE_METHOD='optimize' + +REQUIRED_ARGS={ + 'task_instance': str, + 'backbone': str, + 'optimizer': str, + 'learning_rate': float, + 'batch_size': int + } + +OPTIONAL_ARGS={ + 'mix_ratio': str, + 'target_tag': str, + 'reuse_rag': str + } + +TASK_REQUIRED_ARGS={ + 'paradigm': str, + 'reader': str, + 'train_file': str + } + diff --git a/build/lib/paddlepalm/interface.py b/build/lib/paddlepalm/interface.py new file mode 100644 index 0000000..06c93ac --- /dev/null +++ b/build/lib/paddlepalm/interface.py @@ -0,0 +1,173 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""v1.1""" + +class reader(object): + """interface of data manager.""" + + def __init__(self, config): + assert isinstance(config, dict) + + # @property + # def inputs_attr(self): + # """描述reader输入对象的属性，包含各个对象的名字、shape以及数据类型。当某个对象为标量数据类型（如str, int, float等）时，shape设置为空列表[]，当某个对象的某个维度长度可变时，shape中的相应维度设置为-1. + # Return: + # dict类型。对各个输入对象的属性描述。例如， + # 对于文本分类任务，可能需要包含输入文本和所属标签的id + # {"text": ([], 'str'), + # "label": ([], 'int')} + # 对于标注任务，可能需要输入词序列和对应的标签 + # {"tokens", ([-1], 'str'), + # "tags", ([-1], 'str')} + # 对于机器阅读理解任务，可能需要包含上下文、问题、回答、答案区域的起止位置等 + # {"paragraph", ([], 'str'), + # "question", ([], 'str'), + # "start_position", ([], 'int') + # """ + # raise NotImplementedError() + + @property + def outputs_attr(self): + """描述reader输出对象（被yield出的对象）的属性，包含各个对象的名字、shape以及数据类型。当某个对象为标量数据类型（如str, int, float等）时，shape设置为空列表[]，当某个对象的某个维度长度可变时，shape中的相应维度设置为-1。 + 注意：当使用mini-batch梯度下降学习策略时，，应为常规的输入对象设置batch_size维度（一般为-1） + Return: + dict类型。对各个输入对象的属性描述。例如， + 对于文本分类和匹配任务，yield的输出内容可能包含如下的对象（下游backbone和task可按需访问其中的对象） + {"token_ids": ([-1, max_len], 'int64'), + "input_ids": ([-1, max_len], 'int64'), + "segment_ids": ([-1, max_len], 'int64'), + "input_mask": ([-1, max_len], 'float32'), + "label": ([-1], 'int')} + """ + raise NotImplementedError() + + # def parse_line(self): + # """框架内部使用字典描述每个样本，字典的key为inputs_attr，value为每个input对应的符合attr描述的值。 + # 该函数负责将文本行解析成符合inputs_attr描述的字典类型的样本。默认的parse_line方法会读取json格式的数据集文件，数据集的每一行为json格式描述的样本。 + # 用户可通过对该方法的继承改写来适配不同格式的数据集，例如csv格式甚至tfrecord文件。 + # """ + # raise NotImplementedError() + # + # def tokenize(self, line): + # """框架中内置了word piece tokenizer等分词器，用户可通过修改tokenizer超参数来制定使用的分词器，若内置的分词器均无法满足需求，用户可通过对该方法的继承改写来自定义分词器。 + # Args: + # - line: a unicode string. + # Return: + # a list of tokens + # """ + # raise NotImplementedError() + + def iterator(self): + """数据集遍历接口，注意，当数据集遍历到尾部时该接口应自动完成指针重置，即重新从数据集头部开始新的遍历。 + Yield: + (dict) elements that meet the requirements in output_templete + """ + raise NotImplementedError() + + @property + def num_examples(self): + """数据集中的样本数量，即每个epoch中iterator所生成的样本数。注意，使用滑动窗口等可能导致数据集样本数发生变化的策略时，该接口应返回runtime阶段的实际样本数。""" + raise NotImplementedError() + + + +class backbone(object): + """interface of backbone model.""" + + def __init__(self, config, phase): + """ + Args: + config: dict类型。描述了 多任务配置文件+预训练模型配置文件 中定义超参数 + phase: str类型。运行阶段，目前支持train和predict + """ + assert isinstance(config, dict) + + @property + def inputs_attr(self): + """描述backbone从reader处需要得到的输入对象的属性，包含各个对象的名字、shape以及数据类型。当某个对象为标量数据类型（如str, int, float等）时，shape设置为空列表[]，当某个对象的某个维度长度可变时，shape中的相应维度设置为-1。 + Return: + dict类型。对各个输入对象的属性描述。例如， + 对于文本分类和匹配任务，bert backbone依赖的reader对象主要包含如下的对象 + {"token_ids": ([-1, max_len], 'int64'), + "input_ids": ([-1, max_len], 'int64'), + "segment_ids": ([-1, max_len], 'int64'), + "input_mask": ([-1, max_len], 'float32')}""" + raise NotImplementedError() + + @property + def outputs_attr(self): + """描述backbone输出对象的属性，包含各个对象的名字、shape以及数据类型。当某个对象为标量数据类型（如str, int, float等）时，shape设置为空列表[]，当某个对象的某个维度长度可变时，shape中的相应维度设置为-1。 + Return: + dict类型。对各个输出对象的属性描述。例如， + 对于文本分类和匹配任务，bert backbone的输出内容可能包含如下的对象 + {"word_emb": ([-1, max_seqlen, word_emb_size], 'float32'), + "sentence_emb": ([-1, hidden_size], 'float32'), + "sim_vec": ([-1, hidden_size], 'float32')}""" + raise NotImplementedError() + + def build(self, inputs): + """建立backbone的计算图。将符合inputs_attr描述的静态图Variable输入映射成符合outputs_attr描述的静态图Variable输出。 + Args: + inputs: dict类型。字典中包含inputs_attr中的对象名到计算图Variable的映射，inputs中至少会包含inputs_attr中定义的对象 + Return: + 需要输出的计算图变量，输出对象会被加入到fetch_list中，从而在每个训练/推理step时得到runtime的计算结果，该计算结果会被传入postprocess方法中供用户处理。 + """ + raise NotImplementedError() + + + + +class task_paradigm(object): + + def __init__(self, config, phase, backbone_config): + """ + config: dict类型。描述了 任务实例(task instance)+多任务配置文件 中定义超参数 + phase: str类型。运行阶段，目前支持train和predict + """ + + @property + def inputs_attrs(self): + """描述task_layer需要从reader, backbone等输入对象集合所读取到的输入对象的属性，第一级key为对象集和的名字，如backbone，reader等（后续会支持更灵活的输入），第二级key为对象集和中各对象的属性，包括对象的名字，shape和dtype。当某个对象为标量数据类型（如str, int, float等）时，shape设置为空列表[]，当某个对象的某个维度长度可变时，shape中的相应维度设置为-1。 + Return: + dict类型。对各个对象集及其输入对象的属性描述。""" + raise NotImplementedError() + + @property + def outputs_attr(self): + """描述task输出对象的属性，包括对象的名字，shape和dtype。输出对象会被加入到fetch_list中，从而在每个训练/推理step时得到runtime的计算结果，该计算结果会被传入postprocess方法中供用户处理。 + 当某个对象为标量数据类型（如str, int, float等）时，shape设置为空列表[]，当某个对象的某个维度长度可变时，shape中的相应维度设置为-1。 + Return: + dict类型。对各个输入对象的属性描述。注意，训练阶段必须包含名为loss的输出对象。 + """ + + raise NotImplementedError() + + def build(self, inputs): + """建立task_layer的计算图。将符合inputs_attrs描述的来自各个对象集的静态图Variables映射成符合outputs_attr描述的静态图Variable输出。 + Args: + inputs: dict类型。字典中包含inputs_attrs中的对象名到计算图Variable的映射，inputs中至少会包含inputs_attr中定义的对象 + Return: + 需要输出的计算图变量，输出对象会被加入到fetch_list中，从而在每个训练/推理step时得到runtime的计算结果，该计算结果会被传入postprocess方法中供用户处理。 + + """ + raise NotImplementedError() + + def postprocess(self, rt_outputs): + """每个训练或推理step后针对当前batch的task_layer的runtime计算结果进行相关后处理。注意，rt_outputs除了包含build方法，还自动包含了loss的计算结果。""" + pass + + def post_postprocess(self, global_buffer): + pass + diff --git a/build/lib/paddlepalm/mtl_controller.py b/build/lib/paddlepalm/mtl_controller.py new file mode 100644 index 0000000..3550024 --- /dev/null +++ b/build/lib/paddlepalm/mtl_controller.py @@ -0,0 +1,717 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import importlib +import multiprocessing +from paddle import fluid +from paddle.fluid import layers +import yaml +import json +import logging +import time +import numpy as np + +from paddlepalm.utils.saver import init_pretraining_params, init_checkpoint +from paddlepalm.utils.config_helper import PDConfig +from paddlepalm.utils.print_helper import print_dict +from paddlepalm.utils.reader_helper import create_net_inputs, create_iterator_fn, create_joint_iterator_fn, merge_input_attrs + +from paddlepalm.default_settings import * +from paddlepalm.task_instance import TaskInstance, check_instances + +DEBUG=False +VERBOSE=0 + +def _get_basename(f): + return os.path.splitext(f)[0] + + +def _get_suffix(f): + return os.path.splitext(f)[-1] + + +def _parse_yaml(f, asdict=True, support_cmd_line=False): + assert os.path.exists(f), "file {} not found.".format(f) + if support_cmd_line: + args = PDConfig(yaml_file=f, fuse_args=True) + args.build() + return args.asdict() if asdict else args + else: + if asdict: + with open(f, "r") as fin: + yaml_config = yaml.load(fin, Loader=yaml.SafeLoader) + return yaml_config + else: + raise NotImplementedError() + + +def _parse_json(f, asdict=True, support_cmd_line=False): + assert os.path.exists(f), "file {} not found.".format(f) + if support_cmd_line: + args = PDConfig(json_file=f, fuse_args=support_cmd_line) + args.build() + return args.asdict() if asdict else args + else: + if asdict: + with open(f, "r") as fin: + config = json.load(fin) + return config + else: + raise NotImplementedError() + + +def _parse_list(string, astype=str): + assert isinstance(string, str), "{} is not a string.".format(string) + if ',' not in string: + return [astype(string)] + string = string.replace(',', ' ') + return [astype(i) for i in string.split()] + + +def _try_float(s): + try: + float(s) + return(float(s)) + except: + return s + + +def _check_conf(conf, checklist=None): + assert isinstance(conf, dict), "{} is not a dict.".format(conf) + ret = {} + for k,v in conf.items(): + if isinstance(v, str): + v = _try_float(v) + ret[k] = v + if checklist is not None: + for k, t in checklist: + assert k in ret, "required argument {} is NOT exist in config file.".format(k) + assert isintance(ret[k], t), "value type of argument {} should be {}".format(k, t) + return ret + + +# TODO: 增加None机制，允许hidden size、batch size和seqlen设置为None +def _check_io(in_attr, out_attr, strict=False, in_name="left", out_name="right"): + for name, attr in in_attr.items(): + assert name in out_attr, in_name+': '+name+' not found in '+out_name + if attr != out_attr[name]: + if strict: + raise ValueError(name+': shape or dtype not consistent!') + else: + logging.warning('{}: shape or dtype not consistent!\n{}:\n{}\n{}:\n{}'.format(name, in_name, attr, out_name, out_attr[name])) + + +def _merge_conf(conf1, conf2, conf1_first=True, strict=False): + assert isinstance(conf1, dict), "{} is not a dict.".format(conf1) + assert isinstance(conf2, dict), "{} is not a dict.".format(conf2) + base_conf = conf2 if conf1_first else conf1 + base_conf = base_conf.copy() + new_conf = conf1 if conf1_first else conf2 + + for k, v in new_conf.items(): + if k in base_conf: + if base_conf[k] != v: + raise Warning("value of argument {} has been updated to {}.".format(k, v)) + else: + if strict: + continue + + base_conf[k] = v + return base_conf + + +def _encode_inputs(inputs, scope_name, sep='/', cand_set=None): + outputs = {} + for k, v in inputs.items(): + if cand_set is not None: + if k in cand_set: + outputs[k] = v + if scope_name+sep+k in cand_set: + outputs[scope_name+sep+k] = v + else: + outputs[scope_name+sep+k] = v + return outputs + + +def _decode_inputs(inputs, scope_name, sep='/', keep_unk_keys=True): + outputs = {} + for name, value in inputs.items(): + # var for backbone are also available to tasks + if keep_unk_keys and sep not in name: + outputs[name] = value + # var for this inst + if name.startswith(scope_name+'/'): + outputs[name[len(scope_name+'/'):]] = value + return outputs + + +def _init_env(use_gpu): + if use_gpu: + place = fluid.CUDAPlace(0) + dev_count = fluid.core.get_cuda_device_count() + else: + place = fluid.CPUPlace() + dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + return fluid.Executor(place), dev_count + + +def _fit_attr(conf, fit_attr, strict=False): + for i, attr in fit_attr.items(): + if i not in conf: + if strict: + raise Exception('Argument {} is required to create a controller.'.format(i)) + else: + continue + conf[i] = attr(conf[i]) + return conf + + +class Controller(object): + + def __init__(self, config=None, task_dir='.', for_train=True): + """ + Args: + config: (str|dict) 字符串类型时，给出yaml格式的config配置文件路径； + """ + + self._for_train = for_train + # default mtl_conf + # if config is None and config_path is None: + # raise ValueError('For config and config_path, at least one of them should be set.') + + if isinstance(config, str): + mtl_conf = _parse_yaml(config, support_cmd_line=True) + # if config is not None: + # mtl_conf = _merge_conf(config, mtl_conf) + else: + mtl_conf = config + + mtl_conf = _check_conf(mtl_conf) + mtl_conf = _fit_attr(mtl_conf, REQUIRED_ARGS, strict=True) + mtl_conf = _fit_attr(mtl_conf, OPTIONAL_ARGS, strict=False) + + exe, dev_count = _init_env(use_gpu=mtl_conf.get('use_gpu', True)) + self.exe = exe + self.dev_count = dev_count + + print_dict(mtl_conf, title='main configuration') + + # parse task instances and target tags + instnames = _parse_list(mtl_conf['task_instance']) + assert len(instnames) == len(set(instnames)), "repeated task_instance is NOT supported." + num_instances = len(instnames) + self.num_instances = num_instances + + instname_to_conf = {} + instname_to_id = {} + for id, instname in enumerate(instnames): + instpath = os.path.join(task_dir, instname+'.yaml') + conf = _parse_yaml(instpath, support_cmd_line=False) + # conf = _check_conf(conf, TASK_INSTANCE_REQUIRED_ARGS) + conf = _check_conf(conf) + temp_conf = _merge_conf(mtl_conf, conf, strict=True) + print_dict(temp_conf, title='{} configuration'.format(instname)) + conf = _merge_conf(mtl_conf, conf) + + instname_to_conf[instname] = conf + instname_to_id[instname] = id + + # create task instances + instances = [] + for name in instnames: + instances.append(TaskInstance(name, instname_to_id[name], instname_to_conf[name])) + + check_instances(instances) + + # parse target_tag + if 'target_tag' in mtl_conf: + target_tag = str(mtl_conf['target_tag']) + tags = _parse_list(target_tag, astype=int) + assert len(tags) == len(instnames), "number of target_tag is NOT consistent with that in task_instance." + for tag, inst in zip(tags, instances): + inst.is_target = tag + else: + tags = [i.is_target for i in instances] + num_targets = sum(tags) + num_auxes = num_instances - num_targets + + # parse mix ratios + if 'mix_ratio' in mtl_conf: + mix_ratio = str(mtl_conf['mix_ratio']) + mrs = _parse_list(mix_ratio, astype=float) + assert len(mrs) == num_instances, "number of mix_ratios is NOT consistent with num_instances." + else: + # TODO: 增加joint training模式，让num_epochs平等的作用于每个instance + mrs = [1.0] * num_instances + + for mr, inst in zip(mrs, instances): + inst.mix_ratio = mr + + # parse task layer reuse tags + instname_to_reusehost = {i:i for i in instnames} + if 'task_reuse_tag' in mtl_conf: + tags = _parse_list(mtl_conf['task_reuse_tag'], astype=int) + assert len(tags) == num_targets, 'number of reuse_tags is NOT consistent with number of instances.' + else: + tags = [] + mapper = {} + for inst in instances: + # 有环则tag_id + 1，否则被mapper shutdown + history = set() + history.add(inst.name) + cur_inst = inst + while True: + # 发现有环 + if cur_inst.task_reuse_scope in history: + mapper[inst.name] = len(tags) + break + # 发现在mapper中 + elif cur_inst.task_reuse_scope in mapper: + mapper[inst.name] = mapper[cur_inst.task_reuse_scope] + break + else: + cur_inst = name_to_instance[cur_inst.task_reuse_scope] + history.add(cur_inst.name) + + tags.append(mapper[inst.name]) + # 注意，上面这段需要做单元测试 + + for i in range(1, num_instances): + for j in range(i): + if tags[i] == tags[j]: + # check paradigm of reused tasks + assert instances[i].task_paradigm == \ + instances[j].task_paradigm, \ + "paradigm of reuse tasks should be consistent" + instances[i].task_reuse_scope = instances[j].name + break + + # parse Reader and Paradigm for each instance + for inst in instances: + reader_name = inst.config['reader'] + reader_mod = importlib.import_module(READER_DIR + '.' + reader_name) + Reader = getattr(reader_mod, 'Reader') + + parad_name = inst.config['paradigm'] + parad_mod = importlib.import_module(PARADIGM_DIR + '.' + parad_name) + Paradigm = getattr(parad_mod, 'TaskParadigm') + + inst.Reader = Reader + inst.Paradigm = Paradigm + + # prepare backbone + if 'backbone_config_path' in mtl_conf: + bb_conf = _parse_json(mtl_conf['backbone_config_path']) + bb_conf = _merge_conf(mtl_conf, bb_conf) + else: + bb_conf = mtl_conf + print_dict(bb_conf, title='backbone configuration'.format(instname)) + + bb_name = mtl_conf['backbone'] + bb_mod = importlib.import_module(BACKBONE_DIR + '.' + bb_name) + Backbone = getattr(bb_mod, 'Model') + + self.instances = instances + self.mrs = mrs + self.Backbone = Backbone + self.bb_conf = bb_conf + self.bb_name = bb_name + + self.has_init_train = False + self.has_init_pred = False + + if self._for_train: + print("initialing for training...") + self._init_train() + self.has_init_train = True + + def _init_train(self): + + instances = self.instances + Backbone = self.Backbone + bb_conf = self.bb_conf + bb_name = self.bb_name + dev_count = self.dev_count + num_instances = len(instances) + mrs = self.mrs + + # set first_target/main task instance + main_inst = None + for inst in instances: + if inst.is_target: + main_inst = inst + inst.is_first_target = True + break + main_conf = main_inst.config + if not os.path.exists(main_conf['save_path']): + os.makedirs(main_conf['save_path']) + + # prepare backbone + train_backbone = Backbone(bb_conf, phase='train') + pred_backbone = Backbone(bb_conf, phase='pred') + + # create reader, task + # then check i/o across reader, backbone and task_layer + task_attrs = [] + pred_task_attrs = [] + for inst in instances: + + train_reader = inst.Reader(inst.config, phase='train') + inst.reader['train'] = train_reader + train_parad = inst.Paradigm(inst.config, phase='train', backbone_config=bb_conf) + inst.task_layer['train'] = train_parad + task_attr_from_reader = _encode_inputs(train_parad.inputs_attrs['reader'], inst.name) + task_attrs.append(task_attr_from_reader) + + _check_io(train_backbone.inputs_attr, train_reader.outputs_attr, in_name=bb_name+'_backbone', out_name='reader.train') + _check_io(train_parad.inputs_attrs['reader'], train_reader.outputs_attr, in_name='task_paradigm.train.reader', out_name='reader.train') + _check_io(train_parad.inputs_attrs['backbone'], train_backbone.outputs_attr, in_name='task_paradigm.train.backbone', out_name=bb_name+'_backbone') + + if inst.is_target: + if 'pred_file' not in inst.config: + inst.config['pred_file'] = '' + pred_reader = inst.Reader(inst.config, phase='pred') + pred_parad = inst.Paradigm(inst.config, phase='pred', backbone_config=bb_conf) + # inst.reader['pred'] = pred_reader # 这里创建的reader是个假reader，只是为了读取output_attr而已，所以不做保存 + inst.task_layer['pred'] = pred_parad + # 框架有巨坑，先这样写吧 + task_attr_from_reader = _encode_inputs(pred_parad.inputs_attrs['reader'], inst.name) + pred_task_attrs.append(task_attr_from_reader) + # task_attr = pred_parad.inputs_attrs['reader'] + _check_io(pred_backbone.inputs_attr, pred_reader.outputs_attr, in_name=bb_name+'_backbone', out_name='reader.pred') + _check_io(pred_parad.inputs_attrs['reader'], pred_reader.outputs_attr, in_name='task_paradigm.pred.reader', out_name='reader.pred') + _check_io(pred_parad.inputs_attrs['backbone'], pred_backbone.outputs_attr, in_name='task_paradigm.pred.backbone', out_name=bb_name+'_backbone') + + # merge reader input attrs from backbone and task_instances + joint_input_names, joint_shape_and_dtypes, name_to_position = merge_input_attrs(train_backbone.inputs_attr, task_attrs) + pred_joint_input_names, pred_joint_shape_and_dtypes, _ = merge_input_attrs(pred_backbone.inputs_attr, pred_task_attrs, insert_taskid=False) + # shapes: [task_id, shapes_of_backbone, shapes_of_inst1, ..., shapes_of_instN] + + if DEBUG: + print('----- for debug -----') + print('joint input names:') + print(joint_input_names) + print('joint input shape and dtypes:') + print(joint_shape_and_dtypes) + + # load data + for inst in instances: + print(inst.name+": preparing data...") + inst.reader['train'].load_data() + + # merge dataset iterators and create net input vars + iterators = [] + prefixes = [] + mrs = [] + for inst in instances: + iterators.append(inst.reader['train'].iterator()) + prefixes.append(inst.name) + mrs.append(inst.mix_ratio) + + joint_iterator_fn = create_joint_iterator_fn(iterators, prefixes, joint_shape_and_dtypes, mrs, name_to_position, dev_count=dev_count, verbose=VERBOSE, batch_size=main_conf['batch_size']) + + input_attrs = [[i, j, k] for i, (j,k) in zip(joint_input_names, joint_shape_and_dtypes)] + pred_input_attrs = [[i, j, k] for i, (j,k) in zip(pred_joint_input_names, pred_joint_shape_and_dtypes)] + net_inputs = create_net_inputs(input_attrs, async=True, iterator_fn=joint_iterator_fn, dev_count=dev_count, n_prefetch=3) + + # build backbone and task layers + # 不指定scope名字会挂，框架有坑 + with fluid.unique_name.guard("backbone-"): + bb_output_vars = train_backbone.build(net_inputs) + # bb_output_vars = train_backbone.build(net_inputs) + assert sorted(bb_output_vars.keys()) == sorted(train_backbone.outputs_attr.keys()) + + # 会挂 + # 这里是否有必要新建一个program？是的，被坑死了 + pred_prog = fluid.Program() + pred_init_prog = fluid.Program() + + train_prog = fluid.default_main_program() + train_init_prog = fluid.default_startup_program() + + with fluid.program_guard(main_program = pred_prog, startup_program = pred_init_prog): + pred_net_inputs = create_net_inputs(pred_input_attrs) + with fluid.unique_name.guard("backbone-"): + pred_bb_output_vars = pred_backbone.build(pred_net_inputs) + + fluid.framework.switch_main_program(train_prog) + fluid.framework.switch_startup_program(train_init_prog) + + # pred_backbone = train_backbone + # pred_bb_output_vars = bb_output_vars + + task_output_vars = {} + for inst in instances: + task_inputs = {'backbone': bb_output_vars} + task_inputs_from_reader = _decode_inputs(net_inputs, inst.name) + task_inputs['reader'] = task_inputs_from_reader + + scope = inst.task_reuse_scope + '/' + with fluid.unique_name.guard(scope): + output_vars = inst.build_task_layer(task_inputs, phase='train') + output_vars = {inst.name+'/'+key: val for key, val in output_vars.items()} + old = len(task_output_vars) # for debug + task_output_vars.update(output_vars) + assert len(task_output_vars) - old == len(output_vars) # for debug + + # # prepare predict vars for saving inference model + if inst.is_target: + + # task_attr = inst.task_layer['pred'].inputs_attrs['reader'] + # _input_names, _shape_and_dtypes, _ = merge_input_attrs(pred_backbone.inputs_attr, task_attr, insert_taskid=False) + # pred_input_attrs = [[i, j, k] for i, (j,k) in zip(_input_names, _shape_and_dtypes)] + + with fluid.program_guard(pred_prog, pred_init_prog): + # pred_net_inputs = create_net_inputs(pred_input_attrs) + # 这里同时建立了pred阶段的backbone计算图，不知道是否会造成额外的显存开销（paddle不会计算运行路径） + cur_inputs = _decode_inputs(pred_net_inputs, inst.name) + inst.pred_input = cur_inputs + pred_task_inputs = {'backbone': pred_bb_output_vars, 'reader': cur_inputs} + scope = inst.task_reuse_scope + '/' + with fluid.unique_name.guard(scope): + inst.build_task_layer(pred_task_inputs, phase='pred') + + + bb_fetches = {k: v.name for k,v in bb_output_vars.items()} + task_fetches = {k: v.name for k,v in task_output_vars.items()} + # fetches = bb_fetches.copy() # 注意！框架在多卡时无法fetch变长维度的tensor，这里加入bb的out后会挂 + # fetches.update(task_fetches) + fetches = task_fetches + fetches['__task_id'] = net_inputs['__task_id'].name + + # compute loss + task_id_var = net_inputs['__task_id'] + task_id_vec = layers.one_hot(task_id_var, num_instances) + losses = fluid.layers.concat([task_output_vars[inst.name+'/loss'] for inst in instances], axis=0) + loss = layers.reduce_sum(task_id_vec * losses) + + main_reader = main_inst.reader['train'] + + num_examples = main_reader.num_examples + for inst in instances: + max_train_steps = int(main_conf['num_epochs']* inst.mix_ratio * num_examples) // main_conf['batch_size'] // dev_count + if inst.is_target: + print('{}: expected train steps {}.'.format(inst.name, max_train_steps)) + inst.steps_pur_epoch = inst.reader['train'].num_examples // main_conf['batch_size'] // dev_count + inst.expected_train_steps = max_train_steps + + global_max_train_steps = int(main_conf['num_epochs'] * num_examples * sum(mrs)) // main_conf['batch_size'] // dev_count + print('Estimated overall train steps {}.'.format(global_max_train_steps)) + + if 'warmup_proportion' in main_conf and main_conf['warmup_proportion'] > 0: + warmup_steps = int(global_max_train_steps * main_conf['warmup_proportion']) + print('Warmup steps: '+str(warmup_steps)) + else: + warmup_steps = 0 + # steps_pur_epoch = num_examples // main_conf['batch_size'] // dev_count + + # build optimizer + # 其实也完全可以支持每个任务用它自己的optimizer + if 'optimizer' in main_conf: + optim_mod = importlib.import_module(OPTIMIZER_DIR + '.' + main_conf['optimizer']) + optimize = getattr(optim_mod, OPTIMIZE_METHOD) + optimize(loss, main_conf, max_train_steps, warmup_steps, fluid.default_main_program()) + + loss.persistable = True + if main_conf.get('use_ema', False): + assert 'ema_decay' in main_conf, "ema_decay should be set when use_ema is enabled." + ema = fluid.optimizer.ExponentialMovingAverage(main_conf['ema_decay']) + ema.update() + + # prepare for train + self.train_backbone = train_backbone + self.train_program = fluid.CompiledProgram(fluid.default_main_program()).with_data_parallel(loss_name=loss.name) + self.saver_program = fluid.default_main_program() + + self.main_inst = main_inst + self.fetches = fetches + self.has_init_train = True + self.has_init_pred = True + # self.max_train_steps = max_train_steps + # self.steps_pur_epoch = steps_pur_epoch + + self.exe.run(fluid.default_startup_program()) + print("\nRandomly initialize parameters...\n") + + def _init_pred(self, instance, infer_model_path): + inst = instance + + pred_backbone = self.Backbone(self.bb_conf, phase='pred') + pred_parad = inst.Paradigm(inst.config, phase='pred', backbone_config=self.bb_conf) + inst.task_layer['pred'] = pred_parad + pred_joint_input_names, pred_joint_shape_and_dtypes, name_to_position = merge_input_attrs( + pred_backbone.inputs_attr, inst.task_layer['pred'].inputs_attrs['reader'], + insert_taskid=False) + + pred_prog = inst.load(infer_model_path) + # pred_prog = fluid.CompiledProgram(pred_prog).with_data_parallel() + if inst.reader['pred'] is None: + pred_reader = inst.Reader(inst.config, phase='pred') + inst.reader['pred'] = pred_reader + return pred_prog + + def load_pretrain(self, pretrain_model_path=None): + # load pretrain model (or ckpt) + if pretrain_model_path is None: + assert 'pretrain_model_path' in self.main_conf, "pretrain_model_path NOT set." + pretrain_model_path = self.main_conf['pretrain_model_path'] + + init_pretraining_params( + self.exe, + pretrain_model_path, + main_program=fluid.default_startup_program()) + + + def train(self): + # TODO: 备份各种配置文件，以便用户断点重新训练以及支持将来的预测 + + if not self.has_init_train: + self._init_train() + self.has_init_train = True + + instances = self.instances + num_instances = self.num_instances + main_inst = self.main_inst + main_conf = main_inst.config + + backbone = self.train_backbone + train_program = self.train_program + saver_program = self.saver_program + fetches = self.fetches + + # max_train_steps = self.max_train_steps + # steps_pur_epoch = self.steps_pur_epoch + + finish = [] + for inst in instances: + if inst.is_target: + finish.append(False) + + def train_finish(): + for inst in instances: + if inst.is_target: + if not inst.train_finish: + return False + return True + + # do training + # loss_fetches = {inst.name+'/loss': inst.task_layer['train'].loss for inst in instances} + # old = len(fetches) # for debug + # fetches.update(loss_fetches) + # assert len(fetches) == old + len(loss_fetches) # for debug and avoid user-caused bug + # assert 'task_id' not in fetches # for debug and avoid user-caused bug + # fetches['task_id'] = task_id_var + fetch_names, fetch_list = zip(*fetches.items()) + + main_step = 0 # only count for main task + global_step = 0 # count for all tasks + epoch = 0 + time_begin = time.time() + backbone_buffer = [] + while not train_finish(): + rt_outputs = self.exe.run(train_program, fetch_list=fetch_list) + rt_outputs = {k:v for k,v in zip(fetch_names, rt_outputs)} + rt_task_id = np.squeeze(rt_outputs['__task_id']).tolist() + assert (not isinstance(rt_task_id, list)) or len(set(rt_task_id)) == 1, rt_task_id + rt_task_id = rt_task_id[0] if isinstance(rt_task_id, list) else rt_task_id + cur_task = instances[rt_task_id] + + backbone_rt_outputs = {k:v for k,v in rt_outputs.items() if '/' not in k} + backbone_buffer.append(backbone.postprocess(backbone_rt_outputs)) + + task_rt_outputs = {k[len(cur_task.name+'/'):]: v for k,v in rt_outputs.items() if k.startswith(cur_task.name+'/')} + instances[rt_task_id].task_layer['train'].postprocess(task_rt_outputs) + + global_step += 1 + # if cur_task.is_target: + cur_task.cur_train_step += 1 + + if global_step % main_conf.get('print_every_n_steps', 5) == 0: + loss = rt_outputs[cur_task.name+'/loss'] + loss = np.mean(np.squeeze(loss)).tolist() + + time_end = time.time() + time_cost = time_end - time_begin + + print("Global step: {}. Task: {}, step {}/{} (epoch {}), loss: {:.3f}, speed: {:.2f} steps/s".format( + global_step, cur_task.name, cur_task.cur_train_step, cur_task.steps_pur_epoch, cur_task.cur_train_epoch, + loss, main_conf.get('print_every_n_steps', 5) / time_cost)) + time_begin = time.time() + + if 'save_every_n_steps' in main_conf and global_step % main_conf['save_every_n_steps'] == 0: + save_path = os.path.join(main_conf['save_path'], + "step_" + str(global_step)) + fluid.io.save_persistables(self.exe, save_path, saver_program) + + save_path = os.path.join(main_conf['save_path'], + "step_" + str(global_step) + "_final") + fluid.io.save_persistables(self.exe, save_path, saver_program) + + def pred(self, task_instance, inference_model_dir=None): + if self._for_train: + raise Exception('This controller is a trainer. Please build a new controller with for_train=False for predicting.') + + assert isinstance(task_instance, str) + if isinstance(inference_model_dir, str): + assert os.path.exists(inference_model_dir), inference_model_dir+" not found." + if not self.has_init_pred and inference_model_dir is None: + raise ValueError('infer_model_path is required for prediction.') + + instance = None + for inst in self.instances: + if inst.name == task_instance: + instance = inst + break + + if instance is None: + raise ValueError(task_instance + ' is not a valid task_instance.') + + pred_prog = self._init_pred(instance, inference_model_dir) + + inst = instance + inst.reader['pred'].load_data() + fetch_names, fetch_vars = inst.pred_fetch_list + + # iterator = create_iterator_fn(inst.reader['pred'].iterator, inst.name, pred_joint_shape_and_dtypes, name_to_position) + mapper = {k:v for k,v in inst.pred_input} + buf = [] + for feed in inst.reader['pred'].iterator(): + feed = _encode_inputs(feed, inst.name, cand_set=mapper) + feed = {mapper[k]: v for k,v in feed.items()} + + rt_outputs = self.exe.run(pred_prog, feed, fetch_vars) + rt_outputs = {k:v for k,v in zip(fetch_names, rt_outputs)} + inst.postprocess(rt_outputs, phase='pred') + reader_outputs = inst.reader['pred'].get_epoch_outputs() + inst.epoch_postprocess({'reader':reader_outputs}, phase='pred') + + + + +if __name__ == '__main__': + assert len(sys.argv) == 2, "Usage: python mtl_controller.py " + conf_path = sys.argv[1] + del sys.argv[1] + controller = Controller(conf_path) + if controller.main_conf['do_train']: + controller.train() + + + + + + diff --git a/build/lib/paddlepalm/optimizer/__init__.py b/build/lib/paddlepalm/optimizer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/paddlepalm/optimizer/adam.py b/build/lib/paddlepalm/optimizer/adam.py new file mode 100644 index 0000000..74a0246 --- /dev/null +++ b/build/lib/paddlepalm/optimizer/adam.py @@ -0,0 +1,108 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Optimization and learning rate scheduling.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import paddle.fluid as fluid + +def linear_warmup_decay(learning_rate, warmup_steps, num_train_steps): + """ Applies linear warmup of learning rate from 0 and decay to 0.""" + with fluid.default_main_program()._lr_schedule_guard(): + lr = fluid.layers.tensor.create_global_var( + shape=[1], + value=0.0, + dtype='float32', + persistable=True, + name="scheduled_learning_rate") + + global_step = fluid.layers.learning_rate_scheduler._decay_step_counter() + + with fluid.layers.control_flow.Switch() as switch: + with switch.case(global_step < warmup_steps): + warmup_lr = learning_rate * (global_step / warmup_steps) + fluid.layers.tensor.assign(warmup_lr, lr) + with switch.default(): + decayed_lr = fluid.layers.learning_rate_scheduler.polynomial_decay( + learning_rate=learning_rate, + decay_steps=num_train_steps, + end_learning_rate=0.0, + power=1.0, + cycle=False) + fluid.layers.tensor.assign(decayed_lr, lr) + + return lr + + +def optimize(loss, config, max_train_steps=None, warmup_steps=0, train_program=None): + if warmup_steps > 0: + decay_strategy = config.get('lr_scheduler', 'linear_warmup_decay') + if decay_strategy == 'noam_decay': + scheduled_lr = fluid.layers.learning_rate_scheduler\ + .noam_decay(1/(warmup_steps *(config['learning_rate'] ** 2)), + warmup_steps) + elif decay_strategy == 'linear_warmup_decay': + scheduled_lr = linear_warmup_decay(config['learning_rate'], warmup_steps, + max_train_steps) + else: + raise ValueError("Unkown lr_scheduler, should be " + "'noam_decay' or 'linear_warmup_decay'") + optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr) + else: + optimizer = fluid.optimizer.Adam(learning_rate=config['learning_rate']) + scheduled_lr = config['learning_rate'] + + clip_norm_thres = 1.0 + # When using mixed precision training, scale the gradient clip threshold + # by loss_scaling + fluid.clip.set_gradient_clip( + clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=clip_norm_thres)) + + def exclude_from_weight_decay(name): + if name.find("layer_norm") > -1: + return True + bias_suffix = ["_bias", "_b", ".b_0"] + for suffix in bias_suffix: + if name.endswith(suffix): + return True + return False + + param_list = dict() + + for param in train_program.global_block().all_parameters(): + param_list[param.name] = param * 1.0 + param_list[param.name].stop_gradient = True + + _, param_grads = optimizer.minimize(loss) + + for block in fluid.default_main_program().blocks: + for var_name in block.vars: + if var_name.startswith("embedding"): + print(block.vars[var_name]) + + + if config.get('weight_decay', 0) > 0: + for param, grad in param_grads: + if exclude_from_weight_decay(param.name): + continue + with param.block.program._optimized_guard( + [param, grad]), fluid.framework.name_scope("weight_decay"): + updated_param = param - param_list[ + param.name] * config['weight_decay'] * scheduled_lr + fluid.layers.assign(output=param, input=updated_param) + diff --git a/build/lib/paddlepalm/reader/__init__.py b/build/lib/paddlepalm/reader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/paddlepalm/reader/cls4bert.py b/build/lib/paddlepalm/reader/cls4bert.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/paddlepalm/reader/match4ernie.py b/build/lib/paddlepalm/reader/match4ernie.py new file mode 100644 index 0000000..0ef8fde --- /dev/null +++ b/build/lib/paddlepalm/reader/match4ernie.py @@ -0,0 +1,103 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddlepalm.interface import reader +from paddlepalm.reader.utils.reader4ernie import ClassifyReader + +class Reader(reader): + + def __init__(self, config, phase='train', dev_count=1, print_prefix=''): + """ + Args: + phase: train, eval, pred + """ + + self._is_training = phase == 'train' + + reader = ClassifyReader(config['vocab_path'], + max_seq_len=config['max_seq_len'], + do_lower_case=config.get('do_lower_case', False), + for_cn=config.get('for_cn', False), + random_seed=config.get('seed', None)) + self._reader = reader + self._dev_count = dev_count + + self._batch_size = config['batch_size'] + self._max_seq_len = config['max_seq_len'] + if phase == 'train': + self._input_file = config['train_file'] + self._num_epochs = None # 防止iteartor终止 + self._shuffle = config.get('shuffle', False) + self._shuffle_buffer = config.get('shuffle_buffer', 5000) + elif phase == 'eval': + self._input_file = config['dev_file'] + self._num_epochs = 1 + self._shuffle = False + self._batch_size = config.get('pred_batch_size', self._batch_size) + elif phase == 'pred': + self._input_file = config['pred_file'] + self._num_epochs = 1 + self._shuffle = False + self._batch_size = config.get('pred_batch_size', self._batch_size) + + self._phase = phase + # self._batch_size = + self._print_first_n = config.get('print_first_n', 1) + + + @property + def outputs_attr(self): + if self._is_training: + return {"token_ids": [[-1, -1, 1], 'int64'], + "position_ids": [[-1, -1, 1], 'int64'], + "segment_ids": [[-1, -1, 1], 'int64'], + "input_mask": [[-1, -1, 1], 'float32'], + "label_ids": [[-1,1], 'int64'], + "task_ids": [[-1, -1, 1], 'int64'] + } + else: + return {"token_ids": [[-1, -1, 1], 'int64'], + "position_ids": [[-1, -1, 1], 'int64'], + "segment_ids": [[-1, -1, 1], 'int64'], + "task_ids": [[-1, -1, 1], 'int64'], + "input_mask": [[-1, -1, 1], 'float32'] + } + + + def load_data(self): + self._data_generator = self._reader.data_generator(self._input_file, self._batch_size, self._num_epochs, dev_count=self._dev_count, shuffle=self._shuffle, phase=self._phase) + + def iterator(self): + + def list_to_dict(x): + names = ['token_ids', 'segment_ids', 'position_ids', 'task_ids', 'input_mask', + 'label_ids', 'unique_ids'] + outputs = {n: i for n,i in zip(names, x)} + del outputs['unique_ids'] + if not self._is_training: + del outputs['label_ids'] + return outputs + + for batch in self._data_generator(): + yield list_to_dict(batch) + + def get_epoch_outputs(self): + return {'examples': self._reader.get_examples(self._phase), + 'features': self._reader.get_features(self._phase)} + + @property + def num_examples(self): + return self._reader.get_num_examples(phase=self._phase) + diff --git a/build/lib/paddlepalm/reader/mlm.py b/build/lib/paddlepalm/reader/mlm.py new file mode 100644 index 0000000..823e505 --- /dev/null +++ b/build/lib/paddlepalm/reader/mlm.py @@ -0,0 +1,103 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddlepalm.interface import reader +from paddlepalm.reader.utils.reader4ernie import BaseReader + +class Reader(reader): + + def __init__(self, config, phase='train', dev_count=1, print_prefix=''): + """ + Args: + phase: train, eval, pred + """ + + self._is_training = phase == 'train' + + reader = ClassifyReader(config['vocab_path'], + max_seq_len=config['max_seq_len'], + do_lower_case=config.get('do_lower_case', False), + for_cn=config.get('for_cn', False), + random_seed=config.get('seed', None)) + self._reader = reader + self._dev_count = dev_count + + self._batch_size = config['batch_size'] + self._max_seq_len = config['max_seq_len'] + if phase == 'train': + self._input_file = config['train_file'] + self._num_epochs = None # 防止iteartor终止 + self._shuffle = config.get('shuffle', False) + self._shuffle_buffer = config.get('shuffle_buffer', 5000) + elif phase == 'eval': + self._input_file = config['dev_file'] + self._num_epochs = 1 + self._shuffle = False + self._batch_size = config.get('pred_batch_size', self._batch_size) + elif phase == 'pred': + self._input_file = config['pred_file'] + self._num_epochs = 1 + self._shuffle = False + self._batch_size = config.get('pred_batch_size', self._batch_size) + + self._phase = phase + # self._batch_size = + self._print_first_n = config.get('print_first_n', 1) + + + @property + def outputs_attr(self): + if self._is_training: + return {"token_ids": [[-1, -1, 1], 'int64'], + "position_ids": [[-1, -1, 1], 'int64'], + "segment_ids": [[-1, -1, 1], 'int64'], + "input_mask": [[-1, -1, 1], 'float32'], + "label_ids": [[-1,1], 'int64'], + "task_ids": [[-1, -1, 1], 'int64'] + } + else: + return {"token_ids": [[-1, -1, 1], 'int64'], + "position_ids": [[-1, -1, 1], 'int64'], + "segment_ids": [[-1, -1, 1], 'int64'], + "task_ids": [[-1, -1, 1], 'int64'], + "input_mask": [[-1, -1, 1], 'float32'] + } + + + def load_data(self): + self._data_generator = self._reader.data_generator(self._input_file, self._batch_size, self._num_epochs, dev_count=self._dev_count, shuffle=self._shuffle, phase=self._phase) + + def iterator(self): + + def list_to_dict(x): + names = ['token_ids', 'position_ids', 'segment_ids', 'input_mask', + 'task_ids', 'mask_label', 'mask_pos'] + outputs = {n: i for n,i in zip(names, x)} + del outputs['unique_ids'] + if not self._is_training: + del outputs['label_ids'] + return outputs + + for batch in self._data_generator(): + yield list_to_dict(batch) + + def get_epoch_outputs(self): + return {'examples': self._reader.get_examples(self._phase), + 'features': self._reader.get_features(self._phase)} + + @property + def num_examples(self): + return self._reader.get_num_examples(phase=self._phase) + diff --git a/build/lib/paddlepalm/reader/mrc4bert.py b/build/lib/paddlepalm/reader/mrc4bert.py new file mode 100644 index 0000000..ebf9aed --- /dev/null +++ b/build/lib/paddlepalm/reader/mrc4bert.py @@ -0,0 +1,656 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from paddlepalm.interface import reader +from paddlepalm.utils.textprocess_helper import is_whitespace +from paddlepalm.reader.utils.mrqa_helper import MRQAExample, MRQAFeature +import paddlepalm.tokenizer.bert_tokenizer as tokenization + +class Reader(reader): + + def __init__(self, config, phase='train', dev_count=1, print_prefix=''): + """ + Args: + phase: train, eval, pred + """ + + self._is_training = phase == 'train' + + self._tokenizer = tokenization.FullTokenizer( + vocab_file=config['vocab_path'], do_lower_case=config.get('do_lower_case', False)) + self._max_seq_length = config['max_seq_len'] + self._doc_stride = config['doc_stride'] + self._max_query_length = config['max_query_len'] + + if phase == 'train': + self._input_file = config['train_file'] + self._num_epochs = config['num_epochs'] + self._shuffle = config.get('shuffle', False) + self._shuffle_buffer = config.get('shuffle_buffer', 5000) + if phase == 'eval': + self._input_file = config['dev_file'] + self._num_epochs = 1 + self._shuffle = False + elif phase == 'pred': + self._input_file = config['predict_file'] + self._num_epochs = 1 + self._shuffle = False + + # self._batch_size = + self._batch_size = config['batch_size'] + self._pred_batch_size = config.get('pred_batch_size', self._batch_size) + self._print_first_n = config.get('print_first_n', 1) + self._with_negative = config.get('with_negative', False) + self._sample_rate = config.get('sample_rate', 0.02) + + # TODO: without slide window version + self._with_slide_window = config.get('with_slide_window', False) + + self.vocab = self._tokenizer.vocab + self.vocab_size = len(self.vocab) + self.pad_id = self.vocab["[PAD]"] + self.cls_id = self.vocab["[CLS]"] + self.sep_id = self.vocab["[SEP]"] + self.mask_id = self.vocab["[MASK]"] + + self.current_train_example = -1 + self.num_train_examples = -1 + self.current_train_epoch = -1 + + self.n_examples = None + + print(print_prefix + 'reading raw data...') + with open(input_file, "r") as reader: + self.raw_data = json.load(reader)["data"] + print(print_prfix + 'done!') + + @property + def outputs_attr(self): + if self._is_training: + return {"token_ids": [[-1, self.max_seq_len, 1], 'int64'], + "position_ids": [[-1, self.max_seq_len, 1], 'int64'], + "segment_ids": [[-1, self.max_seq_len, 1], 'int64'], + "input_mask": [[-1, self.max_seq_len, 1], 'float32'], + "start_positions": [[-1, self.max_seq_len, 1], 'int64'], + "end_positions": [[-1, self.max_seq_len, 1], 'int64'] + } + else: + return {"token_ids": [[-1, self.max_seq_len, 1], 'int64'], + "position_ids": [[-1, self.max_seq_len, 1], 'int64'], + "segment_ids": [[-1, self.max_seq_len, 1], 'int64'], + "input_mask": [[-1, self.max_seq_len, 1], 'float32'], + "unique_ids": [[-1, 1], 'int64'] + } + + def iterator(self): + + features = [] + for i in self._num_epochs: + if self._is_training: + print(self.print_prefix + '{} epoch {} {}'.format('-'*16, i, '-'*16)) + example_id = 0 + feature_id = 1000000000 + for line in self.train_file: + raw = self.parse_line(line) + + examples = _raw_to_examples(raw['context'], raw['qa_list'], is_training=self._is_training) + for example in examples: + features.extend(_example_to_features(example, example_id, self._tokenizer, \ + self._max_seq_length, self._doc_stride, self._max_query_length, \ + id_offset=1000000000+len(features), is_training=self._is_training)) + if len(features) >= self._batch_size * self._dev_count: + for batch, total_token_num in _features_to_batches( \ + features[:self._batch_size * self._dev_count], \ + batch_size, in_tokens=self._in_tokens): + temp = prepare_batch_data(batch, total_token_num, \ + max_len=self._max_seq_length, voc_size=-1, \ + pad_id=self.pad_id, cls_id=self.cls_id, sep_id=self.sep_id, mask_id=-1, \ + return_input_mask=True, return_max_len=False, return_num_token=False) + if self._is_training: + tok_ids, pos_ids, seg_ids, input_mask, start_positions, end_positions = temp + yield {"token_ids": tok_ids, "position_ids": pos_ids, "segment_ids": seg_ids, "input_mask": input_mask, "start_positions": start_positions, 'end_positions': end_positions} + else: + tok_ids, pos_ids, seg_ids, input_mask, unique_ids = temp + yield {"token_ids": tok_ids, "position_ids": pos_ids, "segment_ids": seg_ids, "input_mask": input_mask, "unique_ids": unique_ids} + + features = features[self._batch_size * self._dev_count:] + example_id += 1 + + # The last batch may be discarded when running with distributed prediction, so we build some fake batches for the last prediction step. + if self._is_training and len(features) > 0: + pred_batches = [] + for batch, total_token_num in _features_to_batches( \ + features[:self._batch_size * self._dev_count], \ + batch_size, in_tokens=self._in_tokens): + pred_batches.append(prepare_batch_data(batch, total_token_num, max_len=self._max_seq_length, voc_size=-1, + pad_id=self.pad_id, cls_id=self.cls_id, sep_id=self.sep_id, mask_id=-1, \ + return_input_mask=True, return_max_len=False, return_num_token=False)) + + fake_batch = pred_batches[-1] + fake_batch = fake_batch[:-1] + [np.array([-1]*len(fake_batch[0]))] + pred_batches = pred_batches + [fake_batch] * (dev_count - len(pred_batches)) + for batch in pred_batches: + yield batch + + @property + def num_examples(self): + if self.n_examples is None: + self.n_examples = _estimate_runtime_examples(self.raw_data, self._sample_rate, self._tokenizer, \ + self._max_seq_length, self._doc_stride, self._max_query_length, \ + remove_impossible_questions=True, filter_invalid_spans=True) + return self.n_examples + # return math.ceil(n_examples * self._num_epochs / float(self._batch_size * self._dev_count)) + + + +def _raw_to_examples(context, qa_list, is_training=True, remove_impossible_questions=True, filter_invalid_spans=True): + """ + Args: + context: (str) the paragraph that provide information for QA + qa_list: (list) nested dict. Each element in qa_list should contain at least 'id' and 'question'. And the .... + """ + examples = [] + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + for c in context: + if is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + for qa in qa_list: + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + end_position = None + orig_answer_text = None + is_impossible = False + if is_training: + + assert len(qa["answers"]) == 1, "For training, each question should have exactly 1 answer." + + if ('is_impossible' in qa) and (qa["is_impossible"]): + if remove_impossible_questions or filter_invalid_spans: + continue + else: + start_position = -1 + end_position = -1 + orig_answer_text = "" + is_impossible = True + else: + answer = qa["answers"][0] + orig_answer_text = answer["text"] + answer_offset = answer["answer_start"] + answer_length = len(orig_answer_text) + start_position = char_to_word_offset[answer_offset] + end_position = char_to_word_offset[answer_offset + + answer_length - 1] + + # remove corrupt samples + actual_text = " ".join(doc_tokens[start_position:( + end_position + 1)]) + cleaned_answer_text = " ".join( + tokenization.whitespace_tokenize(orig_answer_text)) + if actual_text.find(cleaned_answer_text) == -1: + print(self.print_prefix + "Could not find answer: '%s' vs. '%s'", + actual_text, cleaned_answer_text) + continue + + examples.append(MRQAExample( + qas_id=qas_id, + question_text=question_text, + doc_tokens=doc_tokens, + orig_answer_text=orig_answer_text, + start_position=start_position, + end_position=end_position, + is_impossible=is_impossible)) + + return examples + + + + +def _example_to_features(example, example_id, tokenizer, max_seq_length, doc_stride, max_query_length, id_offset, is_training): + + query_tokens = tokenizer.tokenize(example.question_text) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + tok_to_orig_index = [] + orig_to_tok_index = [] + all_doc_tokens = [] + for (i, token) in enumerate(example.doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + sub_tokens = tokenizer.tokenize(token) + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + tok_start_position = None + tok_end_position = None + if is_training and example.is_impossible: + tok_start_position = -1 + tok_end_position = -1 + if is_training and not example.is_impossible: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + (tok_start_position, tok_end_position) = _improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, tokenizer, + example.orig_answer_text) + + # The -3 accounts for [CLS], [SEP] and [SEP] + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + + # We can have documents that are longer than the maximum sequence length. + # To deal with this we do a sliding window approach, where we take chunks + # of the up to our max length with a stride of `doc_stride`. + _DocSpan = collections.namedtuple( # pylint: disable=invalid-name + "DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + token_to_orig_map = {} + token_is_max_context = {} + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in query_tokens: + tokens.append(token) + segment_ids.append(0) + tokens.append("[SEP]") + segment_ids.append(0) + + for i in range(doc_span.length): + split_token_index = doc_span.start + i + token_to_orig_map[len(tokens)] = tok_to_orig_index[ + split_token_index] + + is_max_context = _check_is_max_context( + doc_spans, doc_span_index, split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + #while len(input_ids) < max_seq_length: + # input_ids.append(0) + # input_mask.append(0) + # segment_ids.append(0) + + #assert len(input_ids) == max_seq_length + #assert len(input_mask) == max_seq_length + #assert len(segment_ids) == max_seq_length + + start_position = None + end_position = None + if is_training and not example.is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and + tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + start_position = 0 + end_position = 0 + continue + else: + doc_offset = len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + if is_training and example.is_impossible: + start_position = 0 + end_position = 0 + + def format_print(): + print("*** Example ***") + print("unique_id: %s" % (unique_id)) + print("example_index: %s" % (example_index)) + print("doc_span_index: %s" % (doc_span_index)) + print("tokens: %s" % " ".join( + [tokenization.printable_text(x) for x in tokens])) + print("token_to_orig_map: %s" % " ".join([ + "%d:%d" % (x, y) + for (x, y) in six.iteritems(token_to_orig_map) + ])) + print("token_is_max_context: %s" % " ".join([ + "%d:%s" % (x, y) + for (x, y) in six.iteritems(token_is_max_context) + ])) + print("input_ids: %s" % " ".join([str(x) for x in input_ids])) + print("input_mask: %s" % " ".join([str(x) for x in input_mask])) + print("segment_ids: %s" % + " ".join([str(x) for x in segment_ids])) + if is_training and example.is_impossible: + print("impossible example") + if is_training and not example.is_impossible: + answer_text = " ".join(tokens[start_position:(end_position + + 1)]) + print("start_position: %d" % (start_position)) + print("end_position: %d" % (end_position)) + print("answer: %s" % + (tokenization.printable_text(answer_text))) + + if self._print_first_n > 0: + format_print() + self._print_first_n -= 1 + + features.append(MRQAFeature( + unique_id=id_offset, + example_index=example_id, + doc_span_index=doc_span_index, + tokens=tokens, + token_to_orig_map=token_to_orig_map, + token_is_max_context=token_is_max_context, + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + start_position=start_position, + end_position=end_position, + is_impossible=example.is_impossible)) + + id_offset += 1 + + return features + + +def _features_to_batches(features, batch_size, in_tokens): + batch, total_token_num, max_len = [], 0, 0 + for (index, feature) in enumerate(features): + if phase == 'train': + self.current_train_example = index + 1 + seq_len = len(feature.input_ids) + labels = [feature.unique_id + ] if feature.start_position is None else [ + feature.start_position, feature.end_position + ] + example = [ + feature.input_ids, feature.segment_ids, range(seq_len) + ] + labels + max_len = max(max_len, seq_len) + + if in_tokens: + to_append = (len(batch) + 1) * max_len <= batch_size + else: + to_append = len(batch) < batch_size + + if to_append: + batch.append(example) + total_token_num += seq_len + else: + yield batch, total_token_num + batch, total_token_num, max_len = [example + ], seq_len, seq_len + if len(batch) > 0: + yield batch, total_token_num + + +def _estimate_runtime_examples(data, sample_rate, tokenizer, \ + max_seq_length, doc_stride, max_query_length, \ + remove_impossible_questions=True, filter_invalid_spans=True): + """Count runtime examples which may differ from number of raw samples due to sliding window operation and etc.. + This is useful to get correct warmup steps for training.""" + + assert sample_rate > 0.0 and sample_rate <= 1.0, "sample_rate must be set between 0.0~1.0" + + num_raw_examples = 0 + for entry in data: + for paragraph in entry["paragraphs"]: + paragraph_text = paragraph["context"] + for qa in paragraph["qas"]: + num_raw_examples += 1 + # print("num raw examples:{}".format(num_raw_examples)) + + def is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False + + sampled_examples = [] + first_samp = True + for entry in data: + for paragraph in entry["paragraphs"]: + doc_tokens = None + for qa in paragraph["qas"]: + if not first_samp and random.random() > sample_rate and sample_rate < 1.0: + continue + + if doc_tokens is None: + paragraph_text = paragraph["context"] + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + for c in paragraph_text: + if is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + assert len(qa["answers"]) == 1, "For training, each question should have exactly 1 answer." + + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + end_position = None + orig_answer_text = None + is_impossible = False + + if ('is_impossible' in qa) and (qa["is_impossible"]): + if remove_impossible_questions or filter_invalid_spans: + continue + else: + start_position = -1 + end_position = -1 + orig_answer_text = "" + is_impossible = True + else: + answer = qa["answers"][0] + orig_answer_text = answer["text"] + answer_offset = answer["answer_start"] + answer_length = len(orig_answer_text) + start_position = char_to_word_offset[answer_offset] + end_position = char_to_word_offset[answer_offset + + answer_length - 1] + + # remove corrupt samples + actual_text = " ".join(doc_tokens[start_position:( + end_position + 1)]) + cleaned_answer_text = " ".join( + tokenization.whitespace_tokenize(orig_answer_text)) + if actual_text.find(cleaned_answer_text) == -1: + continue + + example = MRQAExample( + qas_id=qas_id, + question_text=question_text, + doc_tokens=doc_tokens, + orig_answer_text=orig_answer_text, + start_position=start_position, + end_position=end_position, + is_impossible=is_impossible) + + sampled_examples.append(example) + first_samp = False + + + runtime_sample_rate = len(sampled_examples) / float(num_raw_examples) + + runtime_samp_cnt = 0 + + for example in sampled_examples: + query_tokens = tokenizer.tokenize(example.question_text) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + tok_to_orig_index = [] + orig_to_tok_index = [] + all_doc_tokens = [] + for (i, token) in enumerate(example.doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + sub_tokens = tokenizer.tokenize(token) + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + tok_start_position = None + tok_end_position = None + + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + (tok_start_position, tok_end_position) = _improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, tokenizer, + example.orig_answer_text) + + # The -3 accounts for [CLS], [SEP] and [SEP] + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + + _DocSpan = collections.namedtuple( # pylint: disable=invalid-name + "DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + if filter_invalid_spans and not (tok_start_position >= doc_start and tok_end_position <= doc_end): + continue + runtime_samp_cnt += 1 + return int(runtime_samp_cnt/runtime_sample_rate) + + +def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, + orig_answer_text): + """Returns tokenized answer spans that better match the annotated answer.""" + + # The MRQA annotations are character based. We first project them to + # whitespace-tokenized words. But then after WordPiece tokenization, we can + # often find a "better match". For example: + # + # Question: What year was John Smith born? + # Context: The leader was John Smith (1895-1943). + # Answer: 1895 + # + # The original whitespace-tokenized answer will be "(1895-1943).". However + # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match + # the exact answer, 1895. + # + # However, this is not always possible. Consider the following: + # + # Question: What country is the top exporter of electornics? + # Context: The Japanese electronics industry is the lagest in the world. + # Answer: Japan + # + # In this case, the annotator chose "Japan" as a character sub-span of + # the word "Japanese". Since our WordPiece tokenizer does not split + # "Japanese", we just use "Japanese" as the annotation. This is fairly rare + # in MRQA, but does happen. + tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) + + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + + # Because of the sliding window approach taken to scoring documents, a single + # token can appear in multiple documents. E.g. + # Doc: the man went to the store and bought a gallon of milk + # Span A: the man went to the + # Span B: to the store and bought + # Span C: and bought a gallon of + # ... + # + # Now the word 'bought' will have two scores from spans B and C. We only + # want to consider the score with "maximum context", which we define as + # the *minimum* of its left and right context (the *sum* of left and + # right context will always be the same, of course). + # + # In the example the maximum context for 'bought' would be span C since + # it has 1 left context and 3 right context, while span B has 4 left context + # and 0 right context. + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, + num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + diff --git a/build/lib/paddlepalm/reader/mrc4ernie.py b/build/lib/paddlepalm/reader/mrc4ernie.py new file mode 100644 index 0000000..0c550df --- /dev/null +++ b/build/lib/paddlepalm/reader/mrc4ernie.py @@ -0,0 +1,119 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddlepalm.interface import reader +from paddlepalm.reader.utils.reader4ernie import MRCReader + +class Reader(reader): + + def __init__(self, config, phase='train', dev_count=1, print_prefix=''): + """ + Args: + phase: train, eval, pred + """ + + self._is_training = phase == 'train' + + reader = MRCReader(config['vocab_path'], + max_seq_len=config['max_seq_len'], + do_lower_case=config.get('do_lower_case', False), + tokenizer='FullTokenizer', + for_cn=config.get('for_cn', False), + doc_stride=config['doc_stride'], + max_query_length=config['max_query_len'], + random_seed=config.get('seed', None)) + self._reader = reader + self._dev_count = dev_count + + self._batch_size = config['batch_size'] + self._max_seq_len = config['max_seq_len'] + if phase == 'train': + self._input_file = config['train_file'] + # self._num_epochs = config['num_epochs'] + self._num_epochs = None # 防止iteartor终止 + self._shuffle = config.get('shuffle', False) + self._shuffle_buffer = config.get('shuffle_buffer', 5000) + if phase == 'eval': + self._input_file = config['dev_file'] + self._num_epochs = 1 + self._shuffle = False + self._batch_size = config.get('pred_batch_size', self._batch_size) + elif phase == 'pred': + self._input_file = config['pred_file'] + self._num_epochs = 1 + self._shuffle = False + self._batch_size = config.get('pred_batch_size', self._batch_size) + + self._phase = phase + # self._batch_size = + self._print_first_n = config.get('print_first_n', 1) + + # TODO: without slide window version + self._with_slide_window = config.get('with_slide_window', False) + + + @property + def outputs_attr(self): + if self._is_training: + return {"token_ids": [[-1, -1, 1], 'int64'], + "position_ids": [[-1, -1, 1], 'int64'], + "segment_ids": [[-1, -1, 1], 'int64'], + "input_mask": [[-1, -1, 1], 'float32'], + "start_positions": [[-1, 1], 'int64'], + "end_positions": [[-1, 1], 'int64'], + "task_ids": [[-1, -1, 1], 'int64'] + } + else: + return {"token_ids": [[-1, -1, 1], 'int64'], + "position_ids": [[-1, -1, 1], 'int64'], + "segment_ids": [[-1, -1, 1], 'int64'], + "task_ids": [[-1, -1, 1], 'int64'], + "input_mask": [[-1, -1, 1], 'float32'], + "unique_ids": [[-1, 1], 'int64'] + } + + @property + def epoch_outputs_attr(self): + if not self._is_training: + return {"examples": None, + "features": None} + + def load_data(self): + self._data_generator = self._reader.data_generator(self._input_file, self._batch_size, self._num_epochs, dev_count=self._dev_count, shuffle=self._shuffle, phase=self._phase) + + def iterator(self): + + def list_to_dict(x): + names = ['token_ids', 'segment_ids', 'position_ids', 'task_ids', 'input_mask', + 'start_positions', 'end_positions', 'unique_ids'] + outputs = {n: i for n,i in zip(names, x)} + if self._is_training: + del outputs['unique_ids'] + else: + del outputs['start_positions'] + del outputs['end_positions'] + return outputs + + for batch in self._data_generator(): + yield list_to_dict(batch) + + def get_epoch_outputs(self): + return {'examples': self._reader.get_examples(self._phase), + 'features': self._reader.get_features(self._phase)} + + @property + def num_examples(self): + return self._reader.get_num_examples(phase=self._phase) + diff --git a/build/lib/paddlepalm/reader/utils/__init__.py b/build/lib/paddlepalm/reader/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/paddlepalm/reader/utils/batching4bert.py b/build/lib/paddlepalm/reader/utils/batching4bert.py new file mode 100644 index 0000000..daeb25a --- /dev/null +++ b/build/lib/paddlepalm/reader/utils/batching4bert.py @@ -0,0 +1,184 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mask, padding and batching.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import numpy as np + + +def mask(batch_tokens, total_token_num, vocab_size, CLS=1, SEP=2, MASK=3): + """ + Add mask for batch_tokens, return out, mask_label, mask_pos; + Note: mask_pos responding the batch_tokens after padded; + """ + max_len = max([len(sent) for sent in batch_tokens]) + mask_label = [] + mask_pos = [] + prob_mask = np.random.rand(total_token_num) + # Note: the first token is [CLS], so [low=1] + replace_ids = np.random.randint(1, high=vocab_size, size=total_token_num) + pre_sent_len = 0 + prob_index = 0 + for sent_index, sent in enumerate(batch_tokens): + mask_flag = False + prob_index += pre_sent_len + for token_index, token in enumerate(sent): + prob = prob_mask[prob_index + token_index] + if prob > 0.15: + continue + elif 0.03 < prob <= 0.15: + # mask + if token != SEP and token != CLS: + mask_label.append(sent[token_index]) + sent[token_index] = MASK + mask_flag = True + mask_pos.append(sent_index * max_len + token_index) + elif 0.015 < prob <= 0.03: + # random replace + if token != SEP and token != CLS: + mask_label.append(sent[token_index]) + sent[token_index] = replace_ids[prob_index + token_index] + mask_flag = True + mask_pos.append(sent_index * max_len + token_index) + else: + # keep the original token + if token != SEP and token != CLS: + mask_label.append(sent[token_index]) + mask_pos.append(sent_index * max_len + token_index) + pre_sent_len = len(sent) + # ensure at least mask one word in a sentence + while not mask_flag: + token_index = int(np.random.randint(1, high=len(sent) - 1, size=1)) + if sent[token_index] != SEP and sent[token_index] != CLS: + mask_label.append(sent[token_index]) + sent[token_index] = MASK + mask_flag = True + mask_pos.append(sent_index * max_len + token_index) + mask_label = np.array(mask_label).astype("int64").reshape([-1, 1]) + mask_pos = np.array(mask_pos).astype("int64").reshape([-1, 1]) + return batch_tokens, mask_label, mask_pos + + +def prepare_batch_data(insts, + total_token_num, + max_len=None, + voc_size=0, + pad_id=None, + cls_id=None, + sep_id=None, + mask_id=None, + return_input_mask=True, + return_max_len=True, + return_num_token=False): + """ + 1. generate Tensor of data + 2. generate Tensor of position + 3. generate self attention mask, [shape: batch_size * max_len * max_len] + """ + batch_src_ids = [inst[0] for inst in insts] + batch_sent_ids = [inst[1] for inst in insts] + batch_pos_ids = [inst[2] for inst in insts] + labels_list = [] + # compatible with mrqa, whose example includes start/end positions, + # or unique id + for i in range(3, len(insts[0]), 1): + labels = [inst[i] for inst in insts] + labels = np.array(labels).astype("int64").reshape([-1, 1]) + labels_list.append(labels) + # First step: do mask without padding + if mask_id >= 0: + out, mask_label, mask_pos = mask( + batch_src_ids, + total_token_num, + vocab_size=voc_size, + CLS=cls_id, + SEP=sep_id, + MASK=mask_id) + else: + out = batch_src_ids + # Second step: padding + src_id, self_input_mask = pad_batch_data( + out, + max_len=max_len, + pad_idx=pad_id, return_input_mask=True) + pos_id = pad_batch_data( + batch_pos_ids, + max_len=max_len, + pad_idx=pad_id, + return_pos=False, + return_input_mask=False) + sent_id = pad_batch_data( + batch_sent_ids, + max_len=max_len, + pad_idx=pad_id, + return_pos=False, + return_input_mask=False) + if mask_id >= 0: + return_list = [ + src_id, pos_id, sent_id, self_input_mask, mask_label, mask_pos + ] + labels_list + else: + return_list = [src_id, pos_id, sent_id, self_input_mask] + labels_list + return return_list if len(return_list) > 1 else return_list[0] + + +def pad_batch_data(insts, + max_len=None, + pad_idx=0, + return_pos=False, + return_input_mask=False, + return_max_len=False, + return_num_token=False): + """ + Pad the instances to the max sequence length in batch, and generate the + corresponding position data and input mask. + """ + return_list = [] + if max_len is None: + max_len = max(len(inst) for inst in insts) + # Any token included in dict can be used to pad, since the paddings' loss + # will be masked out by weights and make no effect on parameter gradients. + inst_data = np.array([ + list(inst) + list([pad_idx] * (max_len - len(inst))) for inst in insts + ]) + return_list += [inst_data.astype("int64").reshape([-1, max_len, 1])] + # position data + if return_pos: + inst_pos = np.array([ + list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) + for inst in insts + ]) + return_list += [inst_pos.astype("int64").reshape([-1, max_len, 1])] + if return_input_mask: + # This is used to avoid attention on paddings. + input_mask_data = np.array([[1] * len(inst) + [0] * + (max_len - len(inst)) for inst in insts]) + input_mask_data = np.expand_dims(input_mask_data, axis=-1) + return_list += [input_mask_data.astype("float32")] + if return_max_len: + return_list += [max_len] + if return_num_token: + num_token = 0 + for inst in insts: + num_token += len(inst) + return_list += [num_token] + return return_list if len(return_list) > 1 else return_list[0] + + +if __name__ == "__main__": + pass + + diff --git a/build/lib/paddlepalm/reader/utils/batching4ernie.py b/build/lib/paddlepalm/reader/utils/batching4ernie.py new file mode 100644 index 0000000..d3d1357 --- /dev/null +++ b/build/lib/paddlepalm/reader/utils/batching4ernie.py @@ -0,0 +1,175 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mask, padding and batching.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from six.moves import xrange + + +def mask(batch_tokens, + seg_labels, + mask_word_tags, + total_token_num, + vocab_size, + CLS=1, + SEP=2, + MASK=3): + """ + Add mask for batch_tokens, return out, mask_label, mask_pos; + Note: mask_pos responding the batch_tokens after padded; + """ + max_len = max([len(sent) for sent in batch_tokens]) + mask_label = [] + mask_pos = [] + prob_mask = np.random.rand(total_token_num) + # Note: the first token is [CLS], so [low=1] + replace_ids = np.random.randint(1, high=vocab_size, size=total_token_num) + pre_sent_len = 0 + prob_index = 0 + for sent_index, sent in enumerate(batch_tokens): + mask_flag = False + mask_word = mask_word_tags[sent_index] + prob_index += pre_sent_len + if mask_word: + beg = 0 + for token_index, token in enumerate(sent): + seg_label = seg_labels[sent_index][token_index] + if seg_label == 1: + continue + if beg == 0: + if seg_label != -1: + beg = token_index + continue + + prob = prob_mask[prob_index + beg] + if prob > 0.15: + pass + else: + for index in xrange(beg, token_index): + prob = prob_mask[prob_index + index] + base_prob = 1.0 + if index == beg: + base_prob = 0.15 + if base_prob * 0.2 < prob <= base_prob: + mask_label.append(sent[index]) + sent[index] = MASK + mask_flag = True + mask_pos.append(sent_index * max_len + index) + elif base_prob * 0.1 < prob <= base_prob * 0.2: + mask_label.append(sent[index]) + sent[index] = replace_ids[prob_index + index] + mask_flag = True + mask_pos.append(sent_index * max_len + index) + else: + mask_label.append(sent[index]) + mask_pos.append(sent_index * max_len + index) + + if seg_label == -1: + beg = 0 + else: + beg = token_index + else: + for token_index, token in enumerate(sent): + prob = prob_mask[prob_index + token_index] + if prob > 0.15: + continue + elif 0.03 < prob <= 0.15: + # mask + if token != SEP and token != CLS: + mask_label.append(sent[token_index]) + sent[token_index] = MASK + mask_flag = True + mask_pos.append(sent_index * max_len + token_index) + elif 0.015 < prob <= 0.03: + # random replace + if token != SEP and token != CLS: + mask_label.append(sent[token_index]) + sent[token_index] = replace_ids[prob_index + + token_index] + mask_flag = True + mask_pos.append(sent_index * max_len + token_index) + else: + # keep the original token + if token != SEP and token != CLS: + mask_label.append(sent[token_index]) + mask_pos.append(sent_index * max_len + token_index) + + pre_sent_len = len(sent) + + mask_label = np.array(mask_label).astype("int64").reshape([-1, 1]) + mask_pos = np.array(mask_pos).astype("int64").reshape([-1, 1]) + return batch_tokens, mask_label, mask_pos + + +def pad_batch_data(insts, + pad_idx=0, + return_pos=False, + return_input_mask=False, + return_max_len=False, + return_num_token=False, + return_seq_lens=False): + """ + Pad the instances to the max sequence length in batch, and generate the + corresponding position data and attention bias. + """ + return_list = [] + max_len = max(len(inst) for inst in insts) + # Any token included in dict can be used to pad, since the paddings' loss + # will be masked out by weights and make no effect on parameter gradients. + + inst_data = np.array( + [inst + list([pad_idx] * (max_len - len(inst))) for inst in insts]) + return_list += [inst_data.astype("int64").reshape([-1, max_len, 1])] + + # position data + if return_pos: + inst_pos = np.array([ + list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) + for inst in insts + ]) + + return_list += [inst_pos.astype("int64").reshape([-1, max_len, 1])] + + if return_input_mask: + # This is used to avoid attention on paddings. + input_mask_data = np.array([[1] * len(inst) + [0] * + (max_len - len(inst)) for inst in insts]) + input_mask_data = np.expand_dims(input_mask_data, axis=-1) + return_list += [input_mask_data.astype("float32")] + + if return_max_len: + return_list += [max_len] + + if return_num_token: + num_token = 0 + for inst in insts: + num_token += len(inst) + return_list += [num_token] + + if return_seq_lens: + seq_lens = np.array([len(inst) for inst in insts]) + return_list += [seq_lens.astype("int64").reshape([-1, 1])] + + return return_list if len(return_list) > 1 else return_list[0] + + +if __name__ == "__main__": + + pass diff --git a/build/lib/paddlepalm/reader/utils/mlm_batching.py b/build/lib/paddlepalm/reader/utils/mlm_batching.py new file mode 100644 index 0000000..71d4ab9 --- /dev/null +++ b/build/lib/paddlepalm/reader/utils/mlm_batching.py @@ -0,0 +1,175 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mask, padding and batching.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import numpy as np + + +def mask(batch_tokens, total_token_num, vocab_size, CLS=1, SEP=2, MASK=3): + """ + Add mask for batch_tokens, return out, mask_label, mask_pos; + Note: mask_pos responding the batch_tokens after padded; + """ + max_len = max([len(sent) for sent in batch_tokens]) + mask_label = [] + mask_pos = [] + prob_mask = np.random.rand(total_token_num) + # Note: the first token is [CLS], so [low=1] + replace_ids = np.random.randint(1, high=vocab_size, size=total_token_num) + pre_sent_len = 0 + prob_index = 0 + for sent_index, sent in enumerate(batch_tokens): + mask_flag = False + prob_index += pre_sent_len + for token_index, token in enumerate(sent): + prob = prob_mask[prob_index + token_index] + if prob > 0.15: + continue + elif 0.03 < prob <= 0.15: + # mask + if token != SEP and token != CLS: + mask_label.append(sent[token_index]) + sent[token_index] = MASK + mask_flag = True + mask_pos.append(sent_index * max_len + token_index) + elif 0.015 < prob <= 0.03: + # random replace + if token != SEP and token != CLS: + mask_label.append(sent[token_index]) + sent[token_index] = replace_ids[prob_index + token_index] + mask_flag = True + mask_pos.append(sent_index * max_len + token_index) + else: + # keep the original token + if token != SEP and token != CLS: + mask_label.append(sent[token_index]) + mask_pos.append(sent_index * max_len + token_index) + pre_sent_len = len(sent) + # ensure at least mask one word in a sentence + while not mask_flag: + token_index = int(np.random.randint(1, high=len(sent) - 1, size=1)) + if sent[token_index] != SEP and sent[token_index] != CLS: + mask_label.append(sent[token_index]) + sent[token_index] = MASK + mask_flag = True + mask_pos.append(sent_index * max_len + token_index) + mask_label = np.array(mask_label).astype("int64").reshape([-1, 1]) + mask_pos = np.array(mask_pos).astype("int64").reshape([-1, 1]) + return batch_tokens, mask_label, mask_pos + + +def prepare_batch_data(insts, + total_token_num, + max_len=None, + voc_size=0, + pad_id=None, + cls_id=None, + sep_id=None, + mask_id=None, + task_id=0, + return_input_mask=True, + return_max_len=True, + return_num_token=False): + """ + 1. generate Tensor of data + 2. generate Tensor of position + 3. generate self attention mask, [shape: batch_size * max_len * max_len] + """ + batch_src_ids = [inst[0] for inst in insts] + batch_sent_ids = [inst[1] for inst in insts] + batch_pos_ids = [inst[2] for inst in insts] + + # First step: do mask without padding + out, mask_label, mask_pos = mask( + batch_src_ids, + total_token_num, + vocab_size=voc_size, + CLS=cls_id, + SEP=sep_id, + MASK=mask_id) + # Second step: padding + src_id, self_input_mask = pad_batch_data( + out, + max_len=max_len, + pad_idx=pad_id, return_input_mask=True) + pos_id = pad_batch_data( + batch_pos_ids, + max_len=max_len, + pad_idx=pad_id, + return_pos=False, + return_input_mask=False) + sent_id = pad_batch_data( + batch_sent_ids, + max_len=max_len, + pad_idx=pad_id, + return_pos=False, + return_input_mask=False) + task_ids = np.ones_like( + src_id, dtype="int64") * task_id + return_list = [ + src_id, pos_id, sent_id, self_input_mask, task_ids, mask_label, mask_pos + ] + return return_list if len(return_list) > 1 else return_list[0] + + +def pad_batch_data(insts, + max_len=None, + pad_idx=0, + return_pos=False, + return_input_mask=False, + return_max_len=False, + return_num_token=False): + """ + Pad the instances to the max sequence length in batch, and generate the + corresponding position data and input mask. + """ + return_list = [] + if max_len is None: + max_len = max(len(inst) for inst in insts) + # Any token included in dict can be used to pad, since the paddings' loss + # will be masked out by weights and make no effect on parameter gradients. + inst_data = np.array([ + list(inst) + list([pad_idx] * (max_len - len(inst))) for inst in insts + ]) + return_list += [inst_data.astype("int64").reshape([-1, max_len, 1])] + # position data + if return_pos: + inst_pos = np.array([ + list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) + for inst in insts + ]) + return_list += [inst_pos.astype("int64").reshape([-1, max_len, 1])] + if return_input_mask: + # This is used to avoid attention on paddings. + input_mask_data = np.array([[1] * len(inst) + [0] * + (max_len - len(inst)) for inst in insts]) + input_mask_data = np.expand_dims(input_mask_data, axis=-1) + return_list += [input_mask_data.astype("float32")] + if return_max_len: + return_list += [max_len] + if return_num_token: + num_token = 0 + for inst in insts: + num_token += len(inst) + return_list += [num_token] + return return_list if len(return_list) > 1 else return_list[0] + + +if __name__ == "__main__": + pass + + diff --git a/build/lib/paddlepalm/reader/utils/mrqa_helper.py b/build/lib/paddlepalm/reader/utils/mrqa_helper.py new file mode 100644 index 0000000..e4f8bf5 --- /dev/null +++ b/build/lib/paddlepalm/reader/utils/mrqa_helper.py @@ -0,0 +1,84 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class MRQAExample(object): + """A single training/test example for simple sequence classification. + + For examples without an answer, the start and end position are -1. + """ + + def __init__(self, + qas_id, + question_text, + doc_tokens, + orig_answer_text=None, + start_position=None, + end_position=None, + is_impossible=False): + self.qas_id = qas_id + self.question_text = question_text + self.doc_tokens = doc_tokens + self.orig_answer_text = orig_answer_text + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + def __str__(self): + return self.__repr__() + + def __repr__(self): + s = "" + s += "qas_id: %s" % (tokenization.printable_text(self.qas_id)) + s += ", question_text: %s" % ( + tokenization.printable_text(self.question_text)) + s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) + if self.start_position: + s += ", start_position: %d" % (self.start_position) + if self.start_position: + s += ", end_position: %d" % (self.end_position) + if self.start_position: + s += ", is_impossible: %r" % (self.is_impossible) + return s + + +class MRQAFeature(object): + """A single set of features of data.""" + + def __init__(self, + unique_id, + example_index, + doc_span_index, + tokens, + token_to_orig_map, + token_is_max_context, + input_ids, + input_mask, + segment_ids, + start_position=None, + end_position=None, + is_impossible=None): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tokens = tokens + self.token_to_orig_map = token_to_orig_map + self.token_is_max_context = token_is_max_context + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + diff --git a/build/lib/paddlepalm/reader/utils/reader4ernie.py b/build/lib/paddlepalm/reader/utils/reader4ernie.py new file mode 100644 index 0000000..85a42d6 --- /dev/null +++ b/build/lib/paddlepalm/reader/utils/reader4ernie.py @@ -0,0 +1,989 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import absolute_import + +import sys +import os +import json +import random +import logging +import numpy as np +import six +from io import open +from collections import namedtuple + +import paddlepalm.tokenizer.ernie_tokenizer as tokenization +from paddlepalm.reader.utils.batching4ernie import pad_batch_data +from paddlepalm.reader.utils.mlm_batching import prepare_batch_data + + +log = logging.getLogger(__name__) + +if six.PY3: + import io + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') + sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8') + + +def csv_reader(fd, delimiter='\t'): + def gen(): + for i in fd: + slots = i.rstrip('\n').split(delimiter) + if len(slots) == 1: + yield slots, + else: + yield slots + return gen() + + +class BaseReader(object): + def __init__(self, + vocab_path, + label_map_config=None, + max_seq_len=512, + do_lower_case=True, + in_tokens=False, + is_inference=False, + random_seed=None, + tokenizer="FullTokenizer", + is_classify=True, + is_regression=False, + for_cn=True, + task_id=0): + self.max_seq_len = max_seq_len + self.tokenizer = tokenization.FullTokenizer( + vocab_file=vocab_path, do_lower_case=do_lower_case) + self.vocab = self.tokenizer.vocab + self.pad_id = self.vocab["[PAD]"] + self.cls_id = self.vocab["[CLS]"] + self.sep_id = self.vocab["[SEP]"] + self.in_tokens = in_tokens + self.is_inference = is_inference + self.for_cn = for_cn + self.task_id = task_id + + np.random.seed(random_seed) + + self.is_classify = is_classify + self.is_regression = is_regression + self.current_example = 0 + self.current_epoch = 0 + self.num_examples = 0 + + self.examples = {} + + if label_map_config: + with open(label_map_config, encoding='utf8') as f: + self.label_map = json.load(f) + else: + self.label_map = None + + def get_train_progress(self): + """Gets progress for training phase.""" + return self.current_example, self.current_epoch + + def _read_tsv(self, input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, 'r', encoding='utf8') as f: + reader = csv_reader(f) + headers = next(reader) + Example = namedtuple('Example', headers) + + examples = [] + for line in reader: + example = Example(*line) + examples.append(example) + return examples + + def _truncate_seq_pair(self, tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + def _convert_example_to_record(self, example, max_seq_length, tokenizer): + """Converts a single `Example` into a single `Record`.""" + + text_a = tokenization.convert_to_unicode(example.text_a) + tokens_a = tokenizer.tokenize(text_a) + tokens_b = None + + has_text_b = False + if isinstance(example, dict): + has_text_b = "text_b" in example.keys() + else: + has_text_b = "text_b" in example._fields + + if has_text_b: + text_b = tokenization.convert_to_unicode(example.text_b) + tokens_b = tokenizer.tokenize(text_b) + + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP] with "- 3" + self._truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) + else: + # Account for [CLS] and [SEP] with "- 2" + if len(tokens_a) > max_seq_length - 2: + tokens_a = tokens_a[0:(max_seq_length - 2)] + + # The convention in BERT/ERNIE is: + # (a) For sequence pairs: + # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + # (b) For single sequences: + # tokens: [CLS] the dog is hairy . [SEP] + # type_ids: 0 0 0 0 0 0 0 + # + # Where "type_ids" are used to indicate whether this is the first + # sequence or the second sequence. The embedding vectors for `type=0` and + # `type=1` were learned during pre-training and are added to the wordpiece + # embedding vector (and position vector). This is not *strictly* necessary + # since the [SEP] token unambiguously separates the sequences, but it makes + # it easier for the model to learn the concept of sequences. + # + # For classification tasks, the first vector (corresponding to [CLS]) is + # used as as the "sentence vector". Note that this only makes sense because + # the entire model is fine-tuned. + tokens = [] + text_type_ids = [] + tokens.append("[CLS]") + text_type_ids.append(0) + for token in tokens_a: + tokens.append(token) + text_type_ids.append(0) + tokens.append("[SEP]") + text_type_ids.append(0) + + if tokens_b: + for token in tokens_b: + tokens.append(token) + text_type_ids.append(1) + tokens.append("[SEP]") + text_type_ids.append(1) + + token_ids = tokenizer.convert_tokens_to_ids(tokens) + position_ids = list(range(len(token_ids))) + + if self.is_inference: + Record = namedtuple('Record', + ['token_ids', 'text_type_ids', 'position_ids']) + record = Record( + token_ids=token_ids, + text_type_ids=text_type_ids, + position_ids=position_ids) + else: + if self.label_map: + label_id = self.label_map[example.label] + else: + label_id = example.label + + Record = namedtuple('Record', [ + 'token_ids', 'text_type_ids', 'position_ids', 'label_id', 'qid' + ]) + + qid = None + if "qid" in example._fields: + qid = example.qid + + record = Record( + token_ids=token_ids, + text_type_ids=text_type_ids, + position_ids=position_ids, + label_id=label_id, + qid=qid) + return record + + def _prepare_batch_data(self, examples, batch_size, phase=None): + """generate batch records""" + batch_records, max_len = [], 0 + for index, example in enumerate(examples): + if phase == "train": + self.current_example = index + record = self._convert_example_to_record(example, self.max_seq_len, + self.tokenizer) + max_len = max(max_len, len(record.token_ids)) + if self.in_tokens: + to_append = (len(batch_records) + 1) * max_len <= batch_size + else: + to_append = len(batch_records) < batch_size + if to_append: + batch_records.append(record) + else: + yield self._pad_batch_records(batch_records) + batch_records, max_len = [record], len(record.token_ids) + + if phase == 'pred' and batch_records: + print('the last batch yielded.') + yield self._pad_batch_records(batch_records) + + def get_num_examples(self, input_file=None, phase=None): + if self.examples is not None: + if phase is None: + phase = 'all' + return len(self.examples[phase]) + else: + assert input_file is not None, "Argument input_file should be given or the data_generator should be created when this func is called." + examples = self._read_tsv(input_file) + return len(examples) + + def data_generator(self, + input_file, + batch_size, + epoch, + dev_count=1, + shuffle=True, + phase=None): + examples = self._read_tsv(input_file) + if phase is None: + phase = 'all' + self.examples[phase] = examples + + def wrapper(): + all_dev_batches = [] + if epoch is None: + num_epochs = 99999999 + else: + num_epochs = epoch + for epoch_index in range(num_epochs): + if phase == "train": + self.current_example = 0 + self.current_epoch = epoch_index + if shuffle: + np.random.shuffle(examples) + + for batch_data in self._prepare_batch_data( + examples, batch_size, phase=phase): + if len(all_dev_batches) < dev_count: + all_dev_batches.append(batch_data) + if len(all_dev_batches) == dev_count: + for batch in all_dev_batches: + yield batch + all_dev_batches = [] + def f(): + for i in wrapper(): + yield i + + # def f(): + # try: + # for i in wrapper(): + # yield i + # except Exception as e: + # import traceback + # traceback.print_exc() + + return f + + +class MaskLMReader(BaseReader): + + def _convert_example_to_record(self, example, max_seq_length, tokenizer): + """Converts a single `Example` into a single `Record`.""" + + text_a = tokenization.convert_to_unicode(example.text_a) + tokens_a = tokenizer.tokenize(text_a) + tokens_b = None + + + has_text_b = False + if isinstance(example, dict): + has_text_b = "text_b" in example.keys() + else: + has_text_b = "text_b" in example._fields + + if has_text_b: + text_b = tokenization.convert_to_unicode(example.text_b) + tokens_b = tokenizer.tokenize(text_b) + + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP] with "- 3" + self._truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) + else: + # Account for [CLS] and [SEP] with "- 2" + if len(tokens_a) > max_seq_length - 2: + tokens_a = tokens_a[0:(max_seq_length - 2)] + + # The convention in BERT/ERNIE is: + # (a) For sequence pairs: + # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + # (b) For single sequences: + # tokens: [CLS] the dog is hairy . [SEP] + # type_ids: 0 0 0 0 0 0 0 + # + # Where "type_ids" are used to indicate whether this is the first + # sequence or the second sequence. The embedding vectors for `type=0` and + # `type=1` were learned during pre-training and are added to the wordpiece + # embedding vector (and position vector). This is not *strictly* necessary + # since the [SEP] token unambiguously separates the sequences, but it makes + # it easier for the model to learn the concept of sequences. + # + # For classification tasks, the first vector (corresponding to [CLS]) is + # used as as the "sentence vector". Note that this only makes sense because + # the entire model is fine-tuned. + tokens = [] + text_type_ids = [] + tokens.append("[CLS]") + text_type_ids.append(0) + for token in tokens_a: + tokens.append(token) + text_type_ids.append(0) + tokens.append("[SEP]") + text_type_ids.append(0) + + if tokens_b: + for token in tokens_b: + tokens.append(token) + text_type_ids.append(1) + tokens.append("[SEP]") + text_type_ids.append(1) + + token_ids = tokenizer.convert_tokens_to_ids(tokens) + position_ids = list(range(len(token_ids))) + + Record = namedtuple('Record', + ['token_ids', 'text_type_ids', 'position_ids']) + record = Record( + token_ids=token_ids, + text_type_ids=text_type_ids, + position_ids=position_ids) + + return record + + def batch_reader(examples, batch_size, in_tokens, phase): + batch, total_token_num, max_len = [], 0, 0 + for e in examples: + token_ids, sent_ids, pos_ids = _convert_example_to_record(e, self.max_seq_len, self.tokenizer) + max_len = max(max_len, len(token_ids)) + if in_tokens: + to_append = (len(batch) + 1) * max_len <= batch_size + else: + to_append = len(batch) < batch_size + if to_append: + batch.append(parsed_line) + total_token_num += len(token_ids) + else: + yield batch, total_token_num + batch, total_token_num, max_len = [parsed_line], len( + token_ids), len(token_ids) + + if len(batch) > 0 and phase == 'pred': + yield batch, total_token_num + + def data_generator(self, + input_file, + batch_size, + epoch, + dev_count=1, + shuffle=True, + phase=None): + examples = self._read_tsv(input_file) + if phase is None: + phase = 'all' + self.examples[phase] = examples + + def wrapper(): + all_dev_batches = [] + if epoch is None: + num_epochs = 99999999 + else: + num_epochs = epoch + for epoch_index in range(num_epochs): + if phase == "train": + self.current_example = 0 + self.current_epoch = epoch_index + if shuffle: + np.random.shuffle(examples) + + all_dev_batches = [] + for batch_data, total_token_num in batch_reader(examples, + self.batch_size, self.in_tokens, phase=phase): + batch_data = prepare_batch_data( + batch_data, + total_token_num, + voc_size=self.voc_size, + pad_id=self.pad_id, + cls_id=self.cls_id, + sep_id=self.sep_id, + mask_id=self.mask_id, + max_len=self.max_seq_len, + return_input_mask=True, + return_max_len=False, + return_num_token=False) + + if len(all_dev_batches) < dev_count: + all_dev_batches.append(batch_data) + if len(all_dev_batches) == dev_count: + for batch in all_dev_batches: + yield batch + all_dev_batches = [] + + return wrapper + + +class ClassifyReader(BaseReader): + def _read_tsv(self, input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, 'r', encoding='utf8') as f: + reader = csv_reader(f) + headers = next(reader) + text_indices = [ + index for index, h in enumerate(headers) if h != "label" + ] + Example = namedtuple('Example', headers) + + examples = [] + for line in reader: + for index, text in enumerate(line): + if index in text_indices: + if self.for_cn: + line[index] = text.replace(' ', '') + else: + line[index] = text + example = Example(*line) + examples.append(example) + return examples + + def _pad_batch_records(self, batch_records): + batch_token_ids = [record.token_ids for record in batch_records] + batch_text_type_ids = [record.text_type_ids for record in batch_records] + batch_position_ids = [record.position_ids for record in batch_records] + + if not self.is_inference: + batch_labels = [record.label_id for record in batch_records] + if self.is_classify: + batch_labels = np.array(batch_labels).astype("int64").reshape( + [-1, 1]) + elif self.is_regression: + batch_labels = np.array(batch_labels).astype("float32").reshape( + [-1, 1]) + + if batch_records[0].qid: + batch_qids = [record.qid for record in batch_records] + batch_qids = np.array(batch_qids).astype("int64").reshape( + [-1, 1]) + else: + batch_qids = np.array([]).astype("int64").reshape([-1, 1]) + + # padding + padded_token_ids, input_mask = pad_batch_data( + batch_token_ids, pad_idx=self.pad_id, return_input_mask=True) + padded_text_type_ids = pad_batch_data( + batch_text_type_ids, pad_idx=self.pad_id) + padded_position_ids = pad_batch_data( + batch_position_ids, pad_idx=self.pad_id) + padded_task_ids = np.ones_like( + padded_token_ids, dtype="int64") * self.task_id + + return_list = [ + padded_token_ids, padded_text_type_ids, padded_position_ids, + padded_task_ids, input_mask + ] + if not self.is_inference: + return_list += [batch_labels, batch_qids] + + return return_list + + +class SequenceLabelReader(BaseReader): + def _pad_batch_records(self, batch_records): + batch_token_ids = [record.token_ids for record in batch_records] + batch_text_type_ids = [record.text_type_ids for record in batch_records] + batch_position_ids = [record.position_ids for record in batch_records] + batch_label_ids = [record.label_ids for record in batch_records] + + # padding + padded_token_ids, input_mask, batch_seq_lens = pad_batch_data( + batch_token_ids, + pad_idx=self.pad_id, + return_input_mask=True, + return_seq_lens=True) + padded_text_type_ids = pad_batch_data( + batch_text_type_ids, pad_idx=self.pad_id) + padded_position_ids = pad_batch_data( + batch_position_ids, pad_idx=self.pad_id) + padded_label_ids = pad_batch_data( + batch_label_ids, pad_idx=len(self.label_map) - 1) + padded_task_ids = np.ones_like( + padded_token_ids, dtype="int64") * self.task_id + + return_list = [ + padded_token_ids, padded_text_type_ids, padded_position_ids, + padded_task_ids, input_mask, padded_label_ids, batch_seq_lens + ] + return return_list + + def _reseg_token_label(self, tokens, labels, tokenizer): + assert len(tokens) == len(labels) + ret_tokens = [] + ret_labels = [] + for token, label in zip(tokens, labels): + sub_token = tokenizer.tokenize(token) + if len(sub_token) == 0: + continue + ret_tokens.extend(sub_token) + if len(sub_token) == 1: + ret_labels.append(label) + continue + + if label == "O" or label.startswith("I-"): + ret_labels.extend([label] * len(sub_token)) + elif label.startswith("B-"): + i_label = "I-" + label[2:] + ret_labels.extend([label] + [i_label] * (len(sub_token) - 1)) + elif label.startswith("S-"): + b_laebl = "B-" + label[2:] + e_label = "E-" + label[2:] + i_label = "I-" + label[2:] + ret_labels.extend([b_laebl] + [i_label] * (len(sub_token) - 2) + [e_label]) + elif label.startswith("E-"): + i_label = "I-" + label[2:] + ret_labels.extend([i_label] * (len(sub_token) - 1) + [label]) + + assert len(ret_tokens) == len(ret_labels) + return ret_tokens, ret_labels + + def _convert_example_to_record(self, example, max_seq_length, tokenizer): + tokens = tokenization.convert_to_unicode(example.text_a).split(u"") + labels = tokenization.convert_to_unicode(example.label).split(u"") + tokens, labels = self._reseg_token_label(tokens, labels, tokenizer) + + if len(tokens) > max_seq_length - 2: + tokens = tokens[0:(max_seq_length - 2)] + labels = labels[0:(max_seq_length - 2)] + + tokens = ["[CLS]"] + tokens + ["[SEP]"] + token_ids = tokenizer.convert_tokens_to_ids(tokens) + position_ids = list(range(len(token_ids))) + text_type_ids = [0] * len(token_ids) + no_entity_id = len(self.label_map) - 1 + label_ids = [no_entity_id] + [ + self.label_map[label] for label in labels + ] + [no_entity_id] + + Record = namedtuple( + 'Record', + ['token_ids', 'text_type_ids', 'position_ids', 'label_ids']) + record = Record( + token_ids=token_ids, + text_type_ids=text_type_ids, + position_ids=position_ids, + label_ids=label_ids) + return record + + +class ExtractEmbeddingReader(BaseReader): + def _pad_batch_records(self, batch_records): + batch_token_ids = [record.token_ids for record in batch_records] + batch_text_type_ids = [record.text_type_ids for record in batch_records] + batch_position_ids = [record.position_ids for record in batch_records] + + # padding + padded_token_ids, input_mask, seq_lens = pad_batch_data( + batch_token_ids, + pad_idx=self.pad_id, + return_input_mask=True, + return_seq_lens=True) + padded_text_type_ids = pad_batch_data( + batch_text_type_ids, pad_idx=self.pad_id) + padded_position_ids = pad_batch_data( + batch_position_ids, pad_idx=self.pad_id) + padded_task_ids = np.ones_like( + padded_token_ids, dtype="int64") * self.task_id + + return_list = [ + padded_token_ids, padded_text_type_ids, padded_position_ids, + padded_task_ids, input_mask, seq_lens + ] + + return return_list + + +class MRCReader(BaseReader): + def __init__(self, + vocab_path, + label_map_config=None, + max_seq_len=512, + do_lower_case=True, + in_tokens=False, + random_seed=None, + tokenizer="FullTokenizer", + is_classify=True, + is_regression=False, + for_cn=True, + task_id=0, + doc_stride=128, + max_query_length=64): + self.max_seq_len = max_seq_len + self.tokenizer = tokenization.FullTokenizer( + vocab_file=vocab_path, do_lower_case=do_lower_case) + self.vocab = self.tokenizer.vocab + self.pad_id = self.vocab["[PAD]"] + self.cls_id = self.vocab["[CLS]"] + self.sep_id = self.vocab["[SEP]"] + self.in_tokens = in_tokens + self.for_cn = for_cn + self.task_id = task_id + self.doc_stride = doc_stride + self.max_query_length = max_query_length + self.examples = {} + self.features = {} + + if random_seed is not None: + np.random.seed(random_seed) + + self.current_example = 0 + self.current_epoch = 0 + self.num_examples = 0 + + self.Example = namedtuple('Example', + ['qas_id', 'question_text', 'doc_tokens', 'orig_answer_text', + 'start_position', 'end_position']) + self.Feature = namedtuple("Feature", ["unique_id", "example_index", "doc_span_index", + "tokens", "token_to_orig_map", "token_is_max_context", + "token_ids", "position_ids", "text_type_ids", + "start_position", "end_position"]) + self.DocSpan = namedtuple("DocSpan", ["start", "length"]) + + def _read_json(self, input_file, is_training): + examples = [] + with open(input_file, "r", encoding='utf8') as f: + input_data = json.load(f)["data"] + for entry in input_data: + for paragraph in entry["paragraphs"]: + paragraph_text = paragraph["context"] + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_pos = None + end_pos = None + orig_answer_text = None + + if is_training: + if len(qa["answers"]) != 1: + raise ValueError( + "For training, each question should have exactly 1 answer." + ) + + answer = qa["answers"][0] + orig_answer_text = answer["text"] + answer_offset = answer["answer_start"] + answer_length = len(orig_answer_text) + doc_tokens = [ + paragraph_text[:answer_offset], + paragraph_text[answer_offset:answer_offset + + answer_length], + paragraph_text[answer_offset + answer_length:] + ] + + start_pos = 1 + end_pos = 1 + + actual_text = " ".join(doc_tokens[start_pos:(end_pos + + 1)]) + if actual_text.find(orig_answer_text) == -1: + log.info("Could not find answer: '%s' vs. '%s'", + actual_text, orig_answer_text) + continue + else: + doc_tokens = tokenization.tokenize_chinese_chars( + paragraph_text) + + example = self.Example( + qas_id=qas_id, + question_text=question_text, + doc_tokens=doc_tokens, + orig_answer_text=orig_answer_text, + start_position=start_pos, + end_position=end_pos) + examples.append(example) + + return examples + + def _improve_answer_span(self, doc_tokens, input_start, input_end, + tokenizer, orig_answer_text): + tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) + + def _check_is_max_context(self, doc_spans, cur_span_index, position): + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, + num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + def _convert_example_to_feature(self, examples, max_seq_length, tokenizer, + is_training): + features = [] + unique_id = 1000000000 + + for (example_index, example) in enumerate(examples): + query_tokens = tokenizer.tokenize(example.question_text) + if len(query_tokens) > self.max_query_length: + query_tokens = query_tokens[0:self.max_query_length] + tok_to_orig_index = [] + orig_to_tok_index = [] + all_doc_tokens = [] + for (i, token) in enumerate(example.doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + sub_tokens = tokenizer.tokenize(token) + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + tok_start_position = None + tok_end_position = None + if is_training: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + (tok_start_position, + tok_end_position) = self._improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, + tokenizer, example.orig_answer_text) + + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(self.DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, self.doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + token_to_orig_map = {} + token_is_max_context = {} + text_type_ids = [] + tokens.append("[CLS]") + text_type_ids.append(0) + for token in query_tokens: + tokens.append(token) + text_type_ids.append(0) + tokens.append("[SEP]") + text_type_ids.append(0) + + for i in range(doc_span.length): + split_token_index = doc_span.start + i + token_to_orig_map[len(tokens)] = tok_to_orig_index[ + split_token_index] + + is_max_context = self._check_is_max_context( + doc_spans, doc_span_index, split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + text_type_ids.append(1) + tokens.append("[SEP]") + text_type_ids.append(1) + + token_ids = tokenizer.convert_tokens_to_ids(tokens) + position_ids = list(range(len(token_ids))) + start_position = None + end_position = None + if is_training: + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and + tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + start_position = 0 + end_position = 0 + else: + doc_offset = len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + feature = self.Feature( + unique_id=unique_id, + example_index=example_index, + doc_span_index=doc_span_index, + tokens=tokens, + token_to_orig_map=token_to_orig_map, + token_is_max_context=token_is_max_context, + token_ids=token_ids, + position_ids=position_ids, + text_type_ids=text_type_ids, + start_position=start_position, + end_position=end_position) + features.append(feature) + + unique_id += 1 + + return features + + def _prepare_batch_data(self, records, batch_size, phase=None): + """generate batch records""" + batch_records, max_len = [], 0 + + for index, record in enumerate(records): + if phase == "train": + self.current_example = index + max_len = max(max_len, len(record.token_ids)) + if self.in_tokens: + to_append = (len(batch_records) + 1) * max_len <= batch_size + else: + to_append = len(batch_records) < batch_size + if to_append: + batch_records.append(record) + else: + yield self._pad_batch_records(batch_records, phase == "train") + batch_records, max_len = [record], len(record.token_ids) + + if phase == 'pred' and batch_records: + yield self._pad_batch_records(batch_records, phase == "train") + + def _pad_batch_records(self, batch_records, is_training): + batch_token_ids = [record.token_ids for record in batch_records] + batch_text_type_ids = [record.text_type_ids for record in batch_records] + batch_position_ids = [record.position_ids for record in batch_records] + if is_training: + batch_start_position = [ + record.start_position for record in batch_records + ] + batch_end_position = [ + record.end_position for record in batch_records + ] + batch_start_position = np.array(batch_start_position).astype( + "int64").reshape([-1, 1]) + batch_end_position = np.array(batch_end_position).astype( + "int64").reshape([-1, 1]) + + else: + batch_size = len(batch_token_ids) + batch_start_position = np.zeros( + shape=[batch_size, 1], dtype="int64") + batch_end_position = np.zeros(shape=[batch_size, 1], dtype="int64") + + batch_unique_ids = [record.unique_id for record in batch_records] + batch_unique_ids = np.array(batch_unique_ids).astype("int64").reshape( + [-1, 1]) + + # padding + padded_token_ids, input_mask = pad_batch_data( + batch_token_ids, pad_idx=self.pad_id, return_input_mask=True) + padded_text_type_ids = pad_batch_data( + batch_text_type_ids, pad_idx=self.pad_id) + padded_position_ids = pad_batch_data( + batch_position_ids, pad_idx=self.pad_id) + padded_task_ids = np.ones_like( + padded_token_ids, dtype="int64") * self.task_id + + return_list = [ + padded_token_ids, padded_text_type_ids, padded_position_ids, + padded_task_ids, input_mask, batch_start_position, + batch_end_position, batch_unique_ids + ] + + return return_list + + def get_num_examples(self, phase): + return len(self.features[phase]) + + def get_features(self, phase): + return self.features[phase] + + def get_examples(self, phase): + return self.examples[phase] + + def data_generator(self, + input_file, + batch_size, + epoch, + dev_count=1, + shuffle=True, + phase=None): + + examples = self.examples.get(phase, None) + features = self.features.get(phase, None) + if not examples: + examples = self._read_json(input_file, phase == "train") + features = self._convert_example_to_feature( + examples, self.max_seq_len, self.tokenizer, phase == "train") + self.examples[phase] = examples + self.features[phase] = features + + def wrapper(): + all_dev_batches = [] + if epoch is None: + num_epochs = 99999999 + else: + num_epochs = epoch + for epoch_index in range(num_epochs): + if phase == "train": + self.current_example = 0 + self.current_epoch = epoch_index + if phase == "train" and shuffle: + np.random.shuffle(features) + + for batch_data in self._prepare_batch_data( + features, batch_size, phase=phase): + if len(all_dev_batches) < dev_count: + all_dev_batches.append(batch_data) + if len(all_dev_batches) == dev_count: + for batch in all_dev_batches: + yield batch + all_dev_batches = [] + + return wrapper + + +if __name__ == '__main__': + pass diff --git a/build/lib/paddlepalm/task_instance.py b/build/lib/paddlepalm/task_instance.py new file mode 100644 index 0000000..70a3388 --- /dev/null +++ b/build/lib/paddlepalm/task_instance.py @@ -0,0 +1,286 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddlepalm.interface import reader as base_reader +from paddlepalm.interface import task_paradigm as base_paradigm +import os +import json +from paddle import fluid + +class TaskInstance(object): + + def __init__(self, name, id, config={}, verbose=True): + self._name = name + self._config = config + self._verbose = verbose + + self._save_infermodel_path = os.path.join(self._config['save_path'], 'infer_model') + self._save_ckpt_path = os.path.join(self._config['save_path'], 'ckpt') + + # following flags can be fetch from instance config file + self._is_target = config.get('is_target', True) + self._first_target = config.get('is_first_target', False) + self._task_reuse_scope = config.get('task_reuse_scope', name) + + self._feeded_var_names = None + self._target_vars = None + + # training process management + self._mix_ratio = None + self._expected_train_steps = None + self._expected_train_epochs = None + self._steps_pur_epoch = None + self._cur_train_epoch = 0 + self._cur_train_step = 0 + self._train_finish = False + + # 存放不同运行阶段（train，eval，pred）的数据集reader，key为phase，value为Reader实例 + self._reader = {'train': None, 'eval': None, 'pred': None} + self._input_layer = None + self._inputname_to_varname = {} + self._task_layer = {'train': None, 'eval': None, 'pred': None} + self._pred_input_name_list = [] + self._pred_input_varname_list = [] + self._pred_fetch_name_list = [] + self._pred_fetch_var_list = [] + + self._Reader = None + self._Paradigm = None + + self._exe = fluid.Executor(fluid.CPUPlace()) + + self._save_protocol = { + 'input_names': 'self._pred_input_name_list', + 'input_varnames': 'self._pred_input_varname_list', + 'fetch_list': 'self._pred_fetch_name_list'} + + + def build_task_layer(self, net_inputs, phase): + output_vars = self._task_layer[phase].build(net_inputs) + if phase == 'pred': + self._pred_fetch_name_list, self._pred_fetch_var_list = zip(*output_vars.items()) + return output_vars + + def postprocess(self, rt_outputs, phase): + return self._task_layer[phase].postprocess(rt_outputs) + + def epoch_postprocess(self, epoch_inputs, phase): + return self._task_layer[phase].epoch_postprocess(epoch_inputs) + + def save(self, suffix=''): + dirpath = self._save_infermodel_path + suffix + self._pred_input_varname_list = [str(i) for i in self._pred_input_varname_list] + + fluid.io.save_inference_model(dirpath, self._pred_input_varname_list, self._pred_fetch_var_list, self._exe) + # fluid.io.save_inference_model(dirpath, self._pred_input_varname_list, self._pred_fetch_var_list, self._exe, params_filename='__params__') + print(self._name + ': inference model saved at ' + dirpath) + + conf = {} + for k, strv in self._save_protocol.items(): + exec('v={}'.format(strv)) + conf[k] = v + with open(os.path.join(dirpath, '__conf__'), 'w') as writer: + writer.write(json.dumps(conf, indent=1)) + + def load(self, infer_model_path=None): + if infer_model_path is None: + infer_model_path = self._save_infermodel_path + for k,v in json.load(open(os.path.join(infer_model_path, '__conf__'))).items(): + strv = self._save_protocol[k] + exec('{}=v'.format(strv)) + pred_prog, self._pred_input_varname_list, self._pred_fetch_var_list = \ + fluid.io.load_inference_model(infer_model_path, self._exe) + # pred_prog, self._pred_input_varname_list, self._pred_fetch_var_list = \ + # fluid.io.load_inference_model(infer_model_path, self._exe, params_filename='__params__') + print(self._name+': inference model loaded from ' + infer_model_path) + return pred_prog + + @property + def name(self): + return self._name + + @property + def Reader(self): + return self._Reader + + @Reader.setter + def Reader(self, cls): + assert base_reader.__name__ == cls.__bases__[-1].__name__, \ + "expect: {}, receive: {}.".format(base_reader.__name__, \ + cls.__bases__[-1].__name__) + self._Reader = cls + + @property + def Paradigm(self): + return self._Paradigm + + @Paradigm.setter + def Paradigm(self, cls): + assert base_paradigm.__name__ == cls.__bases__[-1].__name__, \ + "expect: {}, receive: {}.".format(base_paradigm.__name__, \ + cls.__bases__[-1].__name__) + self._Paradigm = cls + + @property + def config(self): + return self._config + + @property + def reader(self): + return self._reader + + @property + def pred_input(self): + return zip(*[self._pred_input_name_list, self._pred_input_varname_list]) + + @pred_input.setter + def pred_input(self, val): + assert isinstance(val, dict) + self._pred_input_name_list, self._pred_input_varname_list = \ + zip(*[[k, v.name] for k,v in val.items()]) + # print(self._pred_input_name_list) + + @property + def pred_fetch_list(self): + return [self._pred_fetch_name_list, self._pred_fetch_var_list] + + @property + def task_layer(self): + return self._task_layer + + @property + def is_first_target(self): + return self._is_first_target + + @is_first_target.setter + def is_first_target(self, value): + self._is_first_target = bool(value) + if self._is_first_target: + assert self._is_target, "ERROR: only target task could be set as main task." + if self._verbose and self._is_first_target: + print("{}: set as main task".format(self._name)) + + @property + def is_target(self): + if self._is_target is not None: + return self._is_target + else: + raise ValueError("{}: is_target is None".format(self._name)) + + @is_target.setter + def is_target(self, value): + self._is_target = bool(value) + if self._verbose: + if self._is_target: + print('{}: set as target task.'.format(self._name)) + else: + print('{}: set as aux task.'.format(self._name)) + + @property + def mix_ratio(self): + if self._mix_ratio is not None: + return self._mix_ratio + else: + raise ValueError("{}: mix_ratio is None".format(self._name)) + + @mix_ratio.setter + def mix_ratio(self, value): + self._mix_ratio = float(value) + if self._verbose: + print('{}: mix_ratio is set to {}'.format(self._name, self._mix_ratio)) + + @property + def expected_train_steps(self): + return self._expected_train_steps + + @expected_train_steps.setter + def expected_train_steps(self, value): + self._expected_train_steps = value + self._expected_train_epochs = value / float(self._steps_pur_epoch) + + @property + def expected_train_epochs(self): + return self._expected_train_epochs + + @property + def cur_train_epoch(self): + return self._cur_train_epoch + + @cur_train_epoch.setter + def cur_train_epoch(self, value): + self._cur_train_epoch = value + + @property + def cur_train_step(self): + return self._cur_train_step + + @cur_train_step.setter + def cur_train_step(self, value): + self._cur_train_step = value + if self._cur_train_step > self._steps_pur_epoch: + self._cur_train_epoch += 1 + self._cur_train_step = 1 + if self._is_target and self._cur_train_step + self._cur_train_epoch * self._steps_pur_epoch >= self._expected_train_steps: + self._train_finish = True + print(self._name+': train finished!') + self.save() + # fluid.io.save_inference_model(self._save_infermodel_path, ) + + @property + def steps_pur_epoch(self): + return self._steps_pur_epoch + + @steps_pur_epoch.setter + def steps_pur_epoch(self, value): + self._steps_pur_epoch = value + + @property + def train_finish(self): + return self._train_finish + + @property + def task_reuse_scope(self): + if self._task_reuse_scope is not None: + return self._task_reuse_scope + else: + raise ValueError("{}: task_reuse_scope is None".format(self._name)) + + @task_reuse_scope.setter + def task_reuse_scope(self, scope_name): + self._task_reuse_scope = str(scope_name) + if self._verbose: + print('{}: task_reuse_scope is set to {}'.format(self._name, self._task_reuse_scope)) + + + + + + + + + + +def check_instances(insts): + """to check ids, first_target""" + pass + +def _check_ids(): + pass + +def _check_targets(): + pass + +def _check_reuse_scopes(): + pass diff --git a/build/lib/paddlepalm/task_paradigm/__init__.py b/build/lib/paddlepalm/task_paradigm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/paddlepalm/task_paradigm/cls.py b/build/lib/paddlepalm/task_paradigm/cls.py new file mode 100644 index 0000000..f69b817 --- /dev/null +++ b/build/lib/paddlepalm/task_paradigm/cls.py @@ -0,0 +1,60 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.fluid as fluid +from paddlepalm.interface import task_paradigm +from paddle.fluid import layers + +class TaskParadigm(task_paradigm): + ''' + classification + ''' + def __init___(self, config, phase): + self._is_training = phase == 'train' + self.sent_emb_size = config['hidden_size'] + self.num_classes = config['n_classes'] + + @property + def inputs_attrs(self): + return {'bakcbone': {"sentence_emb": [-1, self.sent_emb_size], 'float32']}, + 'reader': {"label_ids": [[-1, 1], 'int64']}} + + @property + def outputs_attrs(self): + if self._is_training: + return {'loss': [[1], 'float32']} + else: + return {'logits': [-1, self.num_classes], 'float32'} + + def build(self, **inputs): + sent_emb = inputs['backbone']['sentence_emb'] + label_ids = inputs['reader']['label_ids'] + + logits = fluid.layers.fc( + input=ent_emb + size=self.num_classes, + param_attr=fluid.ParamAttr( + name="cls_out_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.1)), + bias_attr=fluid.ParamAttr( + name="cls_out_b", initializer=fluid.initializer.Constant(0.))) + + loss = fluid.layers.softmax_with_cross_entropy( + logits=logits, label=label_ids) + loss = layers.mean(loss) + if self._is_training: + return {"loss": loss} + else: + return {"logits":logits} diff --git a/build/lib/paddlepalm/task_paradigm/match.py b/build/lib/paddlepalm/task_paradigm/match.py new file mode 100644 index 0000000..58bbf35 --- /dev/null +++ b/build/lib/paddlepalm/task_paradigm/match.py @@ -0,0 +1,70 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.fluid as fluid +from paddlepalm.interface import task_paradigm +from paddle.fluid import layers + +class TaskParadigm(task_paradigm): + ''' + matching + ''' + def __init__(self, config, phase, backbone_config=None): + self._is_training = phase == 'train' + self._hidden_size = backbone_config['hidden_size'] + + @property + def inputs_attrs(self): + if self._is_training: + reader = {"label_ids": [[-1, 1], 'int64']} + else: + reader = {} + bb = {"sentence_pair_embedding": [[-1, self._hidden_size], 'float32']} + return {'reader': reader, 'backbone': bb} + + @property + def outputs_attrs(self): + if self._is_training: + return {"loss": [[1], 'float32']} + else: + return {"logits": [[-1, 1], 'float32']} + + def build(self, inputs): + if self._is_training: + labels = inputs["reader"]["label_ids"] + cls_feats = inputs["backbone"]["sentence_pair_embedding"] + + cls_feats = fluid.layers.dropout( + x=cls_feats, + dropout_prob=0.1, + dropout_implementation="upscale_in_train") + logits = fluid.layers.fc( + input=cls_feats, + size=2, + param_attr=fluid.ParamAttr( + name="cls_out_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02)), + bias_attr=fluid.ParamAttr( + name="cls_out_b", + initializer=fluid.initializer.Constant(0.))) + + if self._is_training: + ce_loss, probs = fluid.layers.softmax_with_cross_entropy( + logits=logits, label=labels, return_softmax=True) + loss = fluid.layers.mean(x=ce_loss) + return {'loss': loss} + else: + return {'logits': logits} + diff --git a/build/lib/paddlepalm/task_paradigm/mlm.py b/build/lib/paddlepalm/task_paradigm/mlm.py new file mode 100644 index 0000000..817b009 --- /dev/null +++ b/build/lib/paddlepalm/task_paradigm/mlm.py @@ -0,0 +1,111 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.fluid as fluid +from paddlepalm.interface import task_paradigm +from paddle.fluid import layers + +class TaskParadigm(task_paradigm): + ''' + matching + ''' + def __init__(self, config, phase, backbone_config=None): + self._is_training = phase == 'train' + self._hidden_size = backbone_config['hidden_size'] + self._vocab_size = backbone_config['vocab_size'] + self._hidden_act = backbone_config['hidden_act'] + self._initializer_range = backbone_config['initializer_range'] + + @property + def inputs_attrs(self): + if self._is_training: + reader = {"label_ids": [[-1, 1], 'int64']} + else: + reader = {} + bb = {"encoder_outputs": [[-1, self._hidden_size], 'float32']} + return {'reader': reader, 'backbone': bb} + + @property + def outputs_attrs(self): + if self._is_training: + return {"loss": [[1], 'float32']} + else: + return {"logits": [[-1, 1], 'float32']} + + def build(self, inputs): + mask_label = inputs["reader"]["mask_label"] + mask_pos = inputs["reader"]["mask_pos"] + word_emb = inputs["backbone"]["word_embedding"] + enc_out = inputs["backbone"]["encoder_outputs"] + + emb_size = word_emb.shape[-1] + + _param_initializer = fluid.initializer.TruncatedNormal( + scale=self._initializer_range) + + mask_pos = fluid.layers.cast(x=mask_pos, dtype='int32') + + reshaped_emb_out = fluid.layers.reshape( + x=enc_out, shape=[-1, emb_size]) + + # extract masked tokens' feature + mask_feat = fluid.layers.gather(input=reshaped_emb_out, index=mask_pos) + num_seqs = fluid.layers.fill_constant(shape=[1], value=512, dtype='int64') + + # transform: fc + mask_trans_feat = fluid.layers.fc( + input=mask_feat, + size=emb_size, + act=self._hidden_act, + param_attr=fluid.ParamAttr( + name='mask_lm_trans_fc.w_0', + initializer=_param_initializer), + bias_attr=fluid.ParamAttr(name='mask_lm_trans_fc.b_0')) + # transform: layer norm + mask_trans_feat = pre_process_layer( + mask_trans_feat, 'n', name='mask_lm_trans') + + mask_lm_out_bias_attr = fluid.ParamAttr( + name="mask_lm_out_fc.b_0", + initializer=fluid.initializer.Constant(value=0.0)) + + # print fluid.default_main_program().global_block() + + # fc_out = fluid.layers.matmul( + # x=mask_trans_feat, + # y=fluid.default_main_program().global_block().var( + # _word_emb_name), + # transpose_y=True) + + fc_out = fluid.layers.matmul( + x=mask_trans_feat, + y=word_emb, + transpose_y=True) + fc_out += fluid.layers.create_parameter( + shape=[self._vocab_size], + dtype='float32', + attr=mask_lm_out_bias_attr, + is_bias=True) + + mask_lm_loss = fluid.layers.softmax_with_cross_entropy( + logits=fc_out, label=mask_label) + loss = fluid.layers.mean(mask_lm_loss) + + if self._is_training: + return {'loss': loss} + else: + return None + + diff --git a/build/lib/paddlepalm/task_paradigm/mrc.py b/build/lib/paddlepalm/task_paradigm/mrc.py new file mode 100644 index 0000000..1d3642a --- /dev/null +++ b/build/lib/paddlepalm/task_paradigm/mrc.py @@ -0,0 +1,486 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.fluid as fluid +from paddlepalm.interface import task_paradigm +import collections +import numpy as np +import os +import math +import six +import paddlepalm.tokenizer.ernie_tokenizer as tokenization +import json + +RawResult = collections.namedtuple("RawResult", + ["unique_id", "start_logits", "end_logits"]) + +class TaskParadigm(task_paradigm): + """""" + + def __init__(self, config, phase, backbone_config=None): + + self._is_training = phase == 'train' + self._max_sequence_length = config['max_seq_len'] + self._hidden_size = backbone_config['hidden_size'] + self._pred_results = [] + + if phase == 'pred': + self._max_answer_length = config.get('max_answer_len', None) + self._null_score_diff_threshold = config.get('null_score_diff_threshold', 0.0) + self._n_best_size = config.get('n_best_size', 20) + self._pred_output_path = config.get('pred_output_path', None) + self._verbose = config.get('verbose', False) + self._with_negative = config.get('with_negative', False) + self._do_lower_case = config.get('do_lower_case', False) + + + @property + def inputs_attrs(self): + if self._is_training: + reader = {"start_positions": [[-1, 1], 'int64'], + "end_positions": [[-1, 1], 'int64']} + else: + reader = {'unique_ids': [[-1, 1], 'int64']} + bb = {"encoder_outputs": [[-1, -1, self._hidden_size], 'float32']} + return {'reader': reader, 'backbone': bb} + + @property + def epoch_inputs_attrs(self): + if not self._is_training: + from_reader = {'examples': None, 'features': None} + return {'reader': from_reader} + + @property + def outputs_attr(self): + if self._is_training: + return {'loss': [[1], 'float32']} + else: + return {'start_logits': [[-1, -1, 1], 'float32'], + 'end_logits': [[-1, -1, 1], 'float32'], + 'unique_ids': [[-1, 1], 'int64']} + + + def build(self, inputs): + if self._is_training: + start_positions = inputs['reader']['start_positions'] + end_positions = inputs['reader']['end_positions'] + else: + unique_id = inputs['reader']['unique_ids'] + + enc_out = inputs['backbone']['encoder_outputs'] + logits = fluid.layers.fc( + input=enc_out, + size=2, + num_flatten_dims=2, + param_attr=fluid.ParamAttr( + name="cls_squad_out_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02)), + bias_attr=fluid.ParamAttr( + name="cls_squad_out_b", initializer=fluid.initializer.Constant(0.))) + + logits = fluid.layers.transpose(x=logits, perm=[2, 0, 1]) + start_logits, end_logits = fluid.layers.unstack(x=logits, axis=0) + + def _compute_single_loss(logits, positions): + """Compute start/end loss for mrc model""" + loss = fluid.layers.softmax_with_cross_entropy( + logits=logits, label=positions) + loss = fluid.layers.mean(x=loss) + return loss + + if self._is_training: + start_loss = _compute_single_loss(start_logits, start_positions) + end_loss = _compute_single_loss(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2.0 + return {'loss': total_loss} + else: + return {'start_logits': start_logits, + 'end_logits': end_logits, + 'unique_ids': unique_id} + + + def postprocess(self, rt_outputs): + """this func will be called after each step(batch) of training/evaluating/predicting process.""" + if not self._is_training: + unique_ids = np.squeeze(rt_outputs['unique_ids'], -1) + start_logits = rt_outputs['start_logits'] + end_logits = rt_outputs['end_logits'] + for idx in range(len(unique_ids)): + + if unique_ids[idx] < 0: + continue + if len(self._pred_results) % 1000 == 0: + print("Predicting example: {}".format(len(self._pred_results))) + uid = int(unique_ids[idx]) + + s = [float(x) for x in start_logits[idx].flat] + e = [float(x) for x in end_logits[idx].flat] + self._pred_results.append( + RawResult( + unique_id=uid, + start_logits=s, + end_logits=e)) + + def epoch_postprocess(self, post_inputs): + """(optional interface) this func will be called after evaluation/predicting process and each epoch during training process.""" + + if not self._is_training: + if self._pred_output_path is None: + raise ValueError('argument pred_output_path not found in config. Please add it into config dict/file.') + examples = post_inputs['reader']['examples'] + features = post_inputs['reader']['features'] + if not os.path.exists(self._pred_output_path): + os.makedirs(self._pred_output_path) + output_prediction_file = os.path.join(self._pred_output_path, "predictions.json") + output_nbest_file = os.path.join(self._pred_output_path, "nbest_predictions.json") + output_null_log_odds_file = os.path.join(self._pred_output_path, "null_odds.json") + _write_predictions(examples, features, self._pred_results, + self._n_best_size, self._max_answer_length, + self._do_lower_case, output_prediction_file, + output_nbest_file, output_null_log_odds_file, + self._with_negative, + self._null_score_diff_threshold, self._verbose) + + +def _write_predictions(all_examples, all_features, all_results, n_best_size, + max_answer_length, do_lower_case, output_prediction_file, + output_nbest_file, output_null_log_odds_file, + with_negative, null_score_diff_threshold, + verbose): + """Write final predictions to the json file and log-odds of null if needed.""" + print("Writing predictions to: %s" % (output_prediction_file)) + print("Writing nbest to: %s" % (output_nbest_file)) + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", [ + "feature_index", "start_index", "end_index", "start_logit", + "end_logit" + ]) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + min_null_feature_index = 0 # the paragraph slice with min mull score + null_start_logit = 0 # the start logit at the slice with min null score + null_end_logit = 0 # the end logit at the slice with min null score + for (feature_index, feature) in enumerate(features): + result = unique_id_to_result[feature.unique_id] + start_indexes = _get_best_indexes(result.start_logits, n_best_size) + end_indexes = _get_best_indexes(result.end_logits, n_best_size) + # if we could have irrelevant answers, get the min score of irrelevant + if with_negative: + feature_null_score = result.start_logits[0] + result.end_logits[ + 0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = result.start_logits[0] + null_end_logit = result.end_logits[0] + for start_index in start_indexes: + for end_index in end_indexes: + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index >= len(feature.tokens): + continue + if end_index >= len(feature.tokens): + continue + if start_index not in feature.token_to_orig_map: + continue + if end_index not in feature.token_to_orig_map: + continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_logit=result.start_logits[start_index], + end_logit=result.end_logits[end_index])) + + if with_negative: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=0, + end_index=0, + start_logit=null_start_logit, + end_logit=null_end_logit)) + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_logit + x.end_logit), + reverse=True) + + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + if pred.start_index > 0: # this is a non-null prediction + tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1 + )] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + + 1)] + tok_text = " ".join(tok_tokens) + + # De-tokenize WordPieces that have been split off. + tok_text = tok_text.replace(" ##", "") + tok_text = tok_text.replace("##", "") + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + final_text = _get_final_text(tok_text, orig_text, do_lower_case, + verbose) + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_logit=pred.start_logit, + end_logit=pred.end_logit)) + + # if we didn't inlude the empty option in the n-best, inlcude it + if with_negative: + if "" not in seen_predictions: + nbest.append( + _NbestPrediction( + text="", + start_logit=null_start_logit, + end_logit=null_end_logit)) + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction( + text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + # debug + if best_non_null_entry is None: + print("Emmm..., sth wrong") + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = entry.start_logit + output["end_logit"] = entry.end_logit + nbest_json.append(output) + + assert len(nbest_json) >= 1 + + if not with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + # predict "" iff the null score - the score of best non-null > threshold + score_diff = score_null - best_non_null_entry.start_logit - ( + best_non_null_entry.end_logit) + scores_diff_json[example.qas_id] = score_diff + if score_diff > null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + + all_nbest_json[example.qas_id] = nbest_json + + with open(output_prediction_file, "w") as writer: + writer.write(json.dumps(all_predictions, indent=4) + "\n") + + with open(output_nbest_file, "w") as writer: + writer.write(json.dumps(all_nbest_json, indent=4) + "\n") + + if with_negative: + with open(output_null_log_odds_file, "w") as writer: + writer.write(json.dumps(scores_diff_json, indent=4) + "\n") + + +def _get_final_text(pred_text, orig_text, do_lower_case, verbose): + """Project the tokenized prediction back to the original text.""" + + # When we created the data, we kept track of the alignment between original + # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So + # now `orig_text` contains the span of our original text corresponding to the + # span that we predicted. + # + # However, `orig_text` may contain extra characters that we don't want in + # our prediction. + # + # For example, let's say: + # pred_text = steve smith + # orig_text = Steve Smith's + # + # We don't want to return `orig_text` because it contains the extra "'s". + # + # We don't want to return `pred_text` because it's already been normalized + # (the MRQA eval script also does punctuation stripping/lower casing but + # our tokenizer does additional normalization like stripping accent + # characters). + # + # What we really want to return is "Steve Smith". + # + # Therefore, we have to apply a semi-complicated alignment heruistic between + # `pred_text` and `orig_text` to get a character-to-charcter alignment. This + # can fail in certain cases in which case we just return `orig_text`. + + def _strip_spaces(text): + ns_chars = [] + ns_to_s_map = collections.OrderedDict() + for (i, c) in enumerate(text): + if c == " ": + continue + ns_to_s_map[len(ns_chars)] = i + ns_chars.append(c) + ns_text = "".join(ns_chars) + return (ns_text, ns_to_s_map) + + # We first tokenize `orig_text`, strip whitespace from the result + # and `pred_text`, and check if they are the same length. If they are + # NOT the same length, the heuristic has failed. If they are the same + # length, we assume the characters are one-to-one aligned. + tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case) + + tok_text = " ".join(tokenizer.tokenize(orig_text)) + + start_position = tok_text.find(pred_text) + if start_position == -1: + if verbose: + print("Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) + return orig_text + end_position = start_position + len(pred_text) - 1 + + (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) + (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + + if len(orig_ns_text) != len(tok_ns_text): + if verbose: + print("Length not equal after stripping spaces: '%s' vs '%s'", + orig_ns_text, tok_ns_text) + return orig_text + + # We then project the characters in `pred_text` back to `orig_text` using + # the character-to-character alignment. + tok_s_to_ns_map = {} + for (i, tok_index) in six.iteritems(tok_ns_to_s_map): + tok_s_to_ns_map[tok_index] = i + + orig_start_position = None + if start_position in tok_s_to_ns_map: + ns_start_position = tok_s_to_ns_map[start_position] + if ns_start_position in orig_ns_to_s_map: + orig_start_position = orig_ns_to_s_map[ns_start_position] + + if orig_start_position is None: + if verbose: + print("Couldn't map start position") + return orig_text + + orig_end_position = None + if end_position in tok_s_to_ns_map: + ns_end_position = tok_s_to_ns_map[end_position] + if ns_end_position in orig_ns_to_s_map: + orig_end_position = orig_ns_to_s_map[ns_end_position] + + if orig_end_position is None: + if verbose: + print("Couldn't map end position") + return orig_text + + output_text = orig_text[orig_start_position:(orig_end_position + 1)] + return output_text + + +def _get_best_indexes(logits, n_best_size): + """Get the n-best logits from a list.""" + index_and_score = sorted( + enumerate(logits), key=lambda x: x[1], reverse=True) + + best_indexes = [] + for i in range(len(index_and_score)): + if i >= n_best_size: + break + best_indexes.append(index_and_score[i][0]) + return best_indexes + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs + + diff --git a/build/lib/paddlepalm/tokenizer/__init__.py b/build/lib/paddlepalm/tokenizer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/paddlepalm/tokenizer/bert_tokenizer.py b/build/lib/paddlepalm/tokenizer/bert_tokenizer.py new file mode 100644 index 0000000..f4cefd0 --- /dev/null +++ b/build/lib/paddlepalm/tokenizer/bert_tokenizer.py @@ -0,0 +1,374 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import unicodedata +import six + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + fin = open(vocab_file) + for num, line in enumerate(fin): + items = convert_to_unicode(line.strip()).split("\t") + if len(items) > 2: + break + token = items[0] + index = items[1] if len(items) == 2 else num + token = token.strip() + vocab[token] = int(index) + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a peice of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class CharTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in text.lower().split(" "): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + self._never_lowercase = ['[UNK]', '[SEP]', '[PAD]', '[CLS]', '[MASK]'] + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case and token not in self._never_lowercase: + token = token.lower() + token = self._run_strip_accents(token) + if token in self._never_lowercase: + split_tokens.extend([token]) + else: + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/build/lib/paddlepalm/tokenizer/ernie_tokenizer.py b/build/lib/paddlepalm/tokenizer/ernie_tokenizer.py new file mode 100644 index 0000000..2e6b044 --- /dev/null +++ b/build/lib/paddlepalm/tokenizer/ernie_tokenizer.py @@ -0,0 +1,417 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import absolute_import + +from io import open + +import collections +import unicodedata +import six + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + with open(vocab_file, encoding='utf8') as fin: + for num, line in enumerate(fin): + items = convert_to_unicode(line.strip()).split("\t") + if len(items) > 2: + break + token = items[0] + index = items[1] if len(items) == 2 else num + token = token.strip() + vocab[token] = int(index) + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a peice of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class CharTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in text.lower().split(" "): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False + + +def tokenize_chinese_chars(text): + """Adds whitespace around any CJK character.""" + + def _is_chinese_char(cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False + + output = [] + buff = "" + for char in text: + cp = ord(char) + if _is_chinese_char(cp) or _is_whitespace(char): + if buff != "": + output.append(buff) + buff = "" + output.append(char) + else: + buff += char + + if buff != "": + output.append(buff) + + return output diff --git a/build/lib/paddlepalm/utils/__init__.py b/build/lib/paddlepalm/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/paddlepalm/utils/config_helper.py b/build/lib/paddlepalm/utils/config_helper.py new file mode 100644 index 0000000..e00f473 --- /dev/null +++ b/build/lib/paddlepalm/utils/config_helper.py @@ -0,0 +1,311 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import argparse +import json +import yaml +import six +import logging + +logging_only_message = "%(message)s" +logging_details = "%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s" + + +class JsonConfig(object): + """ + A high-level api for handling json configure file. + """ + + def __init__(self, config_path): + self._config_dict = self._parse(config_path) + + def _parse(self, config_path): + try: + with open(config_path) as json_file: + config_dict = json.load(json_file) + assert isinstance(config_dict, dict), "Object in {} is NOT a dict.".format(config_path) + except: + raise IOError("Error in parsing bert model config file '%s'" % + config_path) + else: + return config_dict + + def __getitem__(self, key): + return self._config_dict[key] + + def asdict(self): + return self._config_dict + + def print_config(self): + for arg, value in sorted(six.iteritems(self._config_dict)): + print('%s: %s' % (arg, value)) + print('------------------------------------------------') + + +class ArgumentGroup(object): + def __init__(self, parser, title, des): + self._group = parser.add_argument_group(title=title, description=des) + + def add_arg(self, name, type, default, help, **kwargs): + type = str2bool if type == bool else type + self._group.add_argument( + "--" + name, + default=default, + type=type, + help=help + ' Default: %(default)s.', + **kwargs) + + +class ArgConfig(object): + """ + A high-level api for handling argument configs. + """ + + def __init__(self): + parser = argparse.ArgumentParser() + + train_g = ArgumentGroup(parser, "training", "training options.") + train_g.add_arg("epoch", int, 3, "Number of epoches for fine-tuning.") + train_g.add_arg("learning_rate", float, 5e-5, + "Learning rate used to train with warmup.") + train_g.add_arg( + "lr_scheduler", + str, + "linear_warmup_decay", + "scheduler of learning rate.", + choices=['linear_warmup_decay', 'noam_decay']) + train_g.add_arg("weight_decay", float, 0.01, + "Weight decay rate for L2 regularizer.") + train_g.add_arg( + "warmup_proportion", float, 0.1, + "Proportion of training steps to perform linear learning rate warmup for." + ) + train_g.add_arg("save_steps", int, 1000, + "The steps interval to save checkpoints.") + train_g.add_arg( + "loss_scaling", float, 1.0, + "Loss scaling factor for mixed precision training, only valid when use_fp16 is enabled." + ) + train_g.add_arg("pred_dir", str, None, + "Path to save the prediction results") + + log_g = ArgumentGroup(parser, "logging", "logging related.") + log_g.add_arg("skip_steps", int, 10, + "The steps interval to print loss.") + log_g.add_arg("verbose", bool, False, "Whether to output verbose log.") + + run_type_g = ArgumentGroup(parser, "run_type", "running type options.") + run_type_g.add_arg("use_cuda", bool, True, + "If set, use GPU for training.") + run_type_g.add_arg( + "use_fast_executor", bool, False, + "If set, use fast parallel executor (in experiment).") + run_type_g.add_arg( + "num_iteration_per_drop_scope", int, 1, + "Ihe iteration intervals to clean up temporary variables.") + run_type_g.add_arg("do_train", bool, True, + "Whether to perform training.") + run_type_g.add_arg("do_predict", bool, True, + "Whether to perform prediction.") + + custom_g = ArgumentGroup(parser, "customize", "customized options.") + + self.custom_g = custom_g + + self.parser = parser + + def add_arg(self, name, dtype, default, descrip): + self.custom_g.add_arg(name, dtype, default, descrip) + + def build_conf(self): + return self.parser.parse_args() + + +def str2bool(v): + # because argparse does not support to parse "true, False" as python + # boolean directly + return v.lower() in ("true", "t", "1") + + +def print_arguments(args, log=None): + if not log: + print('----------- Configuration Arguments -----------') + for arg, value in sorted(six.iteritems(vars(args))): + print('%s: %s' % (arg, value)) + print('------------------------------------------------') + else: + log.info('----------- Configuration Arguments -----------') + for arg, value in sorted(six.iteritems(vars(args))): + log.info('%s: %s' % (arg, value)) + log.info('------------------------------------------------') + + +class PDConfig(object): + """ + A high-level API for managing configuration files in PaddlePaddle. + Can jointly work with command-line-arugment, json files and yaml files. + """ + + def __init__(self, json_file=None, yaml_file=None, fuse_args=True): + """ + Init funciton for PDConfig. + json_file: the path to the json configure file. + yaml_file: the path to the yaml configure file. + fuse_args: if fuse the json/yaml configs with argparse. + """ + + if json_file is not None and yaml_file is not None: + raise Warning( + "json_file and yaml_file can not co-exist for now. please only use one configure file type." + ) + return + + self.args = None + self.arg_config = {} + self.json_config = {} + self.yaml_config = {} + + parser = argparse.ArgumentParser() + + self.yaml_g = ArgumentGroup(parser, "yaml", "options from yaml.") + self.json_g = ArgumentGroup(parser, "json", "options from json.") + self.com_g = ArgumentGroup(parser, "custom", "customized options.") + + self.parser = parser + + if json_file is not None: + assert isinstance(json_file, str) + self.load_json(json_file, fuse_args=fuse_args) + + if yaml_file is not None: + assert isinstance(yaml_file, str) or isinstance(yaml_file, list) + self.load_yaml(yaml_file, fuse_args=fuse_args) + + def load_json(self, file_path, fuse_args=True): + + if not os.path.exists(file_path): + raise Warning("the json file %s does not exist." % file_path) + return + + with open(file_path, "r") as fin: + self.json_config = json.loads(fin.read()) + fin.close() + + if fuse_args: + for name in self.json_config: + if not isinstance(self.json_config[name], int) \ + and not isinstance(self.json_config[name], float) \ + and not isinstance(self.json_config[name], str) \ + and not isinstance(self.json_config[name], bool): + + continue + + self.json_g.add_arg(name, + type(self.json_config[name]), + self.json_config[name], + "This is from %s" % file_path) + + def load_yaml(self, file_path_list, fuse_args=True): + + if isinstance(file_path_list, str): + file_path_list = [file_path_list] + for file_path in file_path_list: + if not os.path.exists(file_path): + raise Warning("the yaml file %s does not exist." % file_path) + return + + with open(file_path, "r") as fin: + self.yaml_config = yaml.load(fin, Loader=yaml.SafeLoader) + if fuse_args: + for name in self.yaml_config: + if not isinstance(self.yaml_config[name], int) \ + and not isinstance(self.yaml_config[name], float) \ + and not isinstance(self.yaml_config[name], str) \ + and not isinstance(self.yaml_config[name], bool): + + continue + + self.yaml_g.add_arg(name, + type(self.yaml_config[name]), + self.yaml_config[name], + "This is from %s" % file_path) + + def build(self): + self.args = self.parser.parse_args() + self.arg_config = vars(self.args) + + def asdict(self): + return self.arg_config + + def __add__(self, new_arg): + assert isinstance(new_arg, list) or isinstance(new_arg, tuple) + assert len(new_arg) >= 3 + assert self.args is None + + name = new_arg[0] + dtype = new_arg[1] + dvalue = new_arg[2] + desc = new_arg[3] if len( + new_arg) == 4 else "Description is not provided." + + self.com_g.add_arg(name, dtype, dvalue, desc) + + return self + + def __getattr__(self, name): + if name in self.arg_config: + return self.arg_config[name] + + if name in self.json_config: + return self.json_config[name] + + if name in self.yaml_config: + return self.yaml_config[name] + + raise Warning("The argument %s is not defined." % name) + + def Print(self): + + print("-" * 70) + for name in self.arg_config: + print("{: <25}\t{}".format(str(name), str(self.arg_config[name]))) + + for name in self.json_config: + if name not in self.arg_config: + print("{: <25}\t{}" % + (str(name), str(self.json_config[name]))) + + for name in self.yaml_config: + if name not in self.arg_config: + print("{: <25}\t{}" % + (str(name), str(self.yaml_config[name]))) + + print("-" * 70) + + +if __name__ == "__main__": + pd_config = PDConfig(yaml_file="./test/bert_config.yaml") + pd_config += ("my_age", int, 18, "I am forever 18.") + pd_config.build() + + print(pd_config.do_train) + print(pd_config.hidden_size) + print(pd_config.my_age) diff --git a/build/lib/paddlepalm/utils/print_helper.py b/build/lib/paddlepalm/utils/print_helper.py new file mode 100644 index 0000000..842e779 --- /dev/null +++ b/build/lib/paddlepalm/utils/print_helper.py @@ -0,0 +1,31 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MAXLEN = 70 +def print_dict(dic, title=""): + + if title: + title = ' ' + title + ' ' + left_len = (MAXLEN - len(title)) // 2 + title = '-' * left_len + title + right_len = MAXLEN - len(title) + title = title + '-' * right_len + else: + title = '-' * MAXLEN + print(title) + for name in dic: + print("{: <25}\t{}".format(str(name), str(dic[name]))) + print("") + # print("-" * MAXLEN + '\n') diff --git a/build/lib/paddlepalm/utils/reader_helper.py b/build/lib/paddlepalm/utils/reader_helper.py new file mode 100644 index 0000000..0124e63 --- /dev/null +++ b/build/lib/paddlepalm/utils/reader_helper.py @@ -0,0 +1,226 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import random +import numpy as np +import paddle +from paddle import fluid +from paddle.fluid import layers + + +def _check_and_adapt_shape_dtype(rt_val, attr): + if not isinstance(rt_val, np.ndarray): + rt_val = np.array(rt_val) + assert rt_val.dtype != np.dtype('O'), "yielded data is not a valid tensor(number of elements on some dimension may differ)." + if rt_val.dtype == np.dtype('float64'): + rt_val = rt_val.astype('float32') + + shape, dtype = attr + assert rt_val.dtype == np.dtype(dtype), "yielded data type not consistent with attr settings." + assert len(shape) == rt_val.ndim, "yielded data rank(ndim) not consistent with attr settings." + for rt, exp in zip(rt_val.shape, shape): + if exp is None or exp < 0: + continue + assert rt == exp, "yielded data shape is not consistent with attr settings.\nExpected:{}\nActual:{}".format(exp, rt) + return rt_val + + +def _zero_batch(attrs): + pos_attrs = [] + for shape, dtype in attrs: + pos_shape = [size if size and size > 0 else 1 for size in shape] + pos_attrs.append([pos_shape, dtype]) + + return [np.zeros(shape=shape, dtype=dtype) for shape, dtype in pos_attrs] + + +def _zero_batch_x(attrs, batch_size): + pos_attrs = [] + for shape, dtype in attrs: + # pos_shape = [size if size and size > 0 else 5 for size in shape] + pos_shape = [size for size in shape] + if pos_shape[0] == -1: + pos_shape[0] = batch_size + if pos_shape[1] == -1: + pos_shape[1] = 512 # max seq len + pos_attrs.append([pos_shape, dtype]) + + return [np.zeros(shape=shape, dtype=dtype) for shape, dtype in pos_attrs] + + +def create_net_inputs(input_attrs, async=False, iterator_fn=None, dev_count=1, n_prefetch=1): + inputs = [] + ret = {} + for name, shape, dtype in input_attrs: + p = layers.data(name, shape=shape, dtype=dtype) + ret[name] = p + inputs.append(p) + + if async: + assert iterator_fn is not None, "iterator_fn is needed for building async input layer." + reader = fluid.io.PyReader(inputs, capacity=dev_count*n_prefetch, iterable=False) + reader.decorate_batch_generator(iterator_fn) + reader.start() + + return ret + + +def create_iterator_fn(iterator, iterator_prefix, shape_and_dtypes, outname_to_pos, verbose=0): + + def iterator(): + v = verbose + while True: + results = _zero_batch(shape_and_dtypes) + + outputs = next(iterator) # dict type + prefix = iterator_prefixe + for outname, val in outputs.items(): + task_outname = prefix + '/' + outname + + if outname in outname_to_pos: + idx = outname_to_pos[outname] + val = _check_and_adapt_shape_dtype(val, joint_shape_and_dtypes[idx]) + results[idx] = val + + if task_outname in outname_to_pos: + idx = outname_to_pos[task_outname] + val = _check_and_adapt_shape_dtype(val, joint_shape_and_dtypes[idx]) + results[idx] = val + + yield results + + return iterator + + +def create_joint_iterator_fn(iterators, iterator_prefixes, joint_shape_and_dtypes, mrs, outname_to_pos, dev_count=1, keep_one_task=True, verbose=0, batch_size=None): + """ + joint_shape_and_dtypes: 本质上是根据bb和parad的attr设定的，并且由reader中的attr自动填充-1（可变）维度得到，因此通过与iterator的校验可以完成runtime的batch正确性检查 + """ + + task_ids = range(len(iterators)) + weights = [mr / float(sum(mrs)) for mr in mrs] + if not keep_one_task: + dev_count = 1 + + # build fake batch + # 注意这种方法会导致一个问题，用户将某任务的mix ratio设置成0后，并不能避免从该任务上读数据，若用户将数据集删掉则会导致崩溃；不过相比之前的zero batch方法，这种方法不必作出只能有一个size=-1的维度且第0维的-1必须是batch size的假设 + results = _zero_batch(joint_shape_and_dtypes) + outbuf = {} + for id in task_ids: + outputs = next(iterators[id]) # dict type + outbuf[id] = outputs + prefix = iterator_prefixes[id] + for outname, val in outputs.items(): + task_outname = prefix + '/' + outname + + if outname in outname_to_pos: + idx = outname_to_pos[outname] + val = _check_and_adapt_shape_dtype(val, joint_shape_and_dtypes[idx]) + results[idx] = val + + if task_outname in outname_to_pos: + idx = outname_to_pos[task_outname] + val = _check_and_adapt_shape_dtype(val, joint_shape_and_dtypes[idx]) + results[idx] = val + + fake_batch = results + dev_count_bak = dev_count + + def iterator(): + v = verbose + while True: + id = np.random.choice(task_ids, p=weights) + results = fake_batch + if v > 0: + print('----- debug joint iterator -----') + print('sampled task id: '+str(id)) + task_id_tensor = np.array([[id]]).astype("int64") + results[0] = task_id_tensor + + for i in range(dev_count): + # results = _zero_batch(joint_shape_and_dtypes, batch_size=batch_size) + # results[0] = task_id_tensor + if id in outbuf: + outputs = outbuf[id] + del outbuf[id] + else: + outputs = next(iterators[id]) # dict type + + prefix = iterator_prefixes[id] + for outname, val in outputs.items(): + if v > 0: + print('reader generate: '+outname) + task_outname = prefix + '/' + outname + + if outname in outname_to_pos: + idx = outname_to_pos[outname] + if v > 0: + print(outname + ' is insert in idx ' + str(idx)) + val = _check_and_adapt_shape_dtype(val, joint_shape_and_dtypes[idx]) + results[idx] = val + + if task_outname in outname_to_pos: + idx = outname_to_pos[task_outname] + if v > 0: + print(task_outname + ' is insert in idx ' + str(idx)) + val = _check_and_adapt_shape_dtype(val, joint_shape_and_dtypes[idx]) + results[idx] = val + + if v > 0: + print('yielded batch len and shapes:') + print(len(results)) + for i in results: + print(np.shape(i)) + print('') + v -= 1 + yield results + + return iterator + + +def merge_input_attrs(backbone_attr, task_attrs, insert_taskid=True): + """ + Args: + task_attrs(list[dict]|dict): task input attributes, key=attr_name, val=[shape, dtype], support single task and nested tasks + """ + if isinstance(task_attrs, dict): + task_attrs = [task_attrs] + + if insert_taskid: + ret = [([1,1], 'int64')] + names = ['__task_id'] + start = 1 + else: + ret = [] + names = [] + start = 0 + + names += sorted(backbone_attr.keys()) + ret.extend([backbone_attr[k] for k in names[start:]]) + name_to_position = {} + # pos=0 is for task_id, thus we start from 1 + for pos, k in enumerate(names): + name_to_position[k] = pos + for task_attr in task_attrs: + task_names = sorted(task_attr.keys()) + names.extend(task_names) + ret.extend([task_attr[k] for k in task_names]) + for pos, k in enumerate(task_names, start=len(name_to_position)): + name_to_position[k] = pos + return names, ret, name_to_position + + diff --git a/build/lib/paddlepalm/utils/saver.py b/build/lib/paddlepalm/utils/saver.py new file mode 100644 index 0000000..ffc3f0d --- /dev/null +++ b/build/lib/paddlepalm/utils/saver.py @@ -0,0 +1,65 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +import six +import ast +import copy + +import numpy as np +import paddle.fluid as fluid + +def init_checkpoint(exe, init_checkpoint_path, main_program, skip_list = []): + assert os.path.exists( + init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path + + def existed_persitables(var): + if not fluid.io.is_persistable(var): + return False + if var.name in skip_list: + return False + return os.path.exists(os.path.join(init_checkpoint_path, var.name)) + + fluid.io.load_vars( + exe, + init_checkpoint_path, + main_program=main_program, + predicate=existed_persitables) + print("Load model from {}".format(init_checkpoint_path)) + + +def init_pretraining_params(exe, + pretraining_params_path, + main_program): + assert os.path.exists(pretraining_params_path + ), "[%s] cann't be found." % pretraining_params_path + + def existed_params(var): + if not isinstance(var, fluid.framework.Parameter): + return False + return os.path.exists(os.path.join(pretraining_params_path, var.name)) + + print("Load pretraining parameters from {}...\n".format( + pretraining_params_path)) + + fluid.io.load_vars( + exe, + pretraining_params_path, + main_program=main_program, + predicate=existed_params) + + diff --git a/build/lib/paddlepalm/utils/textprocess_helper.py b/build/lib/paddlepalm/utils/textprocess_helper.py new file mode 100644 index 0000000..35607ec --- /dev/null +++ b/build/lib/paddlepalm/utils/textprocess_helper.py @@ -0,0 +1,19 @@ +# -*- coding: UTF-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False diff --git a/dist/paddle_palm-1.2-py2.7.egg b/dist/paddle_palm-1.2-py2.7.egg new file mode 100644 index 0000000000000000000000000000000000000000..6e95d54e1c862d4ab6c0e77fa25308e3f119fc7f GIT binary patch literal 164003 zcmagE18^=;*EJa1Hg0U&wrx9kV%xTJW81cE+qQ9&+{}GvrvCr^W~OSos;j%YdY|2W z*6Fp^S$z~`K*7*}fPkQYT+B5TVyweF93X&z_K<*pkpEp3laQd3k{4HCP?Y({039lcfxz2drxSZ41IvC5Gd+$8eP)(`E zx9|7veP$wk5|r8_I0|ZE;162?d6l4jLR`#)pXG`NT zxrlL0{Rj?=$IFR8PG5BL;2nfH#fv?>LXjA(0_KYx3W#$r9u3}nW^IVo6kX~E+$GXn z^cQ{i;;03SpAescGFb4^BuF{*5*`?5&tQVtF=C{_*Nm4KH~s+#!V@nk;fNI%XvNCc} zHr7bh*g0poGgnpX)v2dHZ?L!@G7_1xt|CLs?_1q(Ctq5UY|{Dm;yzVHxe zBUtB38!Qnb=IZ>@g+GeWfhs+%=rEB8pXE$Ua1qb620G9sZ~St8AMa`=*m)2b<;=e7 z?t|wq&_;n=pppU=-!B}(6muS6p1 z#>y`0r~}qgUo+@nxGCxRP0h56Tk~#Onj|#yo&j>`kC1%6awpcwD%m!=3IDQ=rGwBV zA>u=d+!uWl$s?1Ypv{{W4L4sI;@vIJKv93y)4}S(N4FOj4_q4clp!D1ei6_UP*N4q z8)NL0wxye#SHGSU3!=CC-|X-p#vv+@lx@LjFw0lwCrtYKLLb!~?>H75H9L;^z~6)S zkSF^2IU?Oj;eJN+Y4_HP6Cay}E$q(3qoavCmKgjdT&i?QzENJB_t`E_68m`OM_al%QR2wR-oMV zYL&tQ?iyHJynH&RJznCZ`kSSi z4w3C6Njm-d-2(Cj(*zv|rG9qT#j#GKoce+kb@N9tTg*;(X9h_+W@%-KtZ+dI zewO35n5ky}Y(11}2Y8wX$~qWwp$^%L;w44YgrYBXCqR2J*RKc%B>%wZT#+71RxxQvz4>)4zBUQ(E16cgOv z1l6;IX)eZ8Lkg1qbF~N)XqA$Zx-G5(Ua21KEaie=gft`Um_Gqm;)&@GbzAVUlNOD% zWhS~U-?&Q-;@;6%oO$~`VT3DlF1lXl_~w}=Y4q{g%{^(&U1#aWWZFjbU2X_j()%|$ zQ!Z$f6^FR>vWDrJ+nL)-SvI7CNdTP9df9;* zI!g}+b6j!6E1YSV@Nv4EevhW6`oT{~{*z;I!RM(+c%a%VVM+cW1(M-FUtX|Q-XjyJm(C}9}?7piJJc5QBIm~m4;>}%s#O}E0Et?ej zZF&CDcrbLy3-@BRD{0Gi5n-I0kYK>=1c6X-Eum}eAT4)7QElM+8}05Jv{LX!%h|ia@%!{{jT zq2VzWs^eqWeYe6oj+~bUgX9%F4dgn8T218|1bi=|8Iay%^b`Cre**WPXk*kfQ|Ei zkU9bZ`rpwani>lK9gY6q(ExiFIxi~+I%gwuv;PtGAI-Ay9CQHx?{3(CJH~%Ux!5}x z*qXVS+5RUPjijv9gcP0R^u)=3rv;BySHRt#Ly-CxsQZ66;9z8GYHQ|TWNXJ@U|z`* z)kJ&uS8-iTXujgxk&w5$=5vS&vi9R;w+u`30X^{&56lFD6x2~7rw?JC@tEY@tyyDJ zo*iKt;M*pEH50`zFrxm$2~a^`PoQX^5Qz1#`dGnG2NWVr&Ug~`Ox!hA4B3haMk?6N z)CjLPEW1)P>)%EMYu%*JMvxLYb$UstlP_CbPX;rtH@yJR{0--1YCcVAJ7&O|j|iKok* zlSU@VXk>wC4n@yFtBj*nhH1C-#DJBbT`k)(@km6?>F(~Q>FHbO82&h-y_;MqBT-$3u&GqT&>G;4@GEwz%V&T|J?5752VS=__ zL((P)a;ZfiO{MYHxeI&zEFpyD%?aq)@pJpjFc6>1`$HN|>yr@dKCr2*^OmHr1x94$ z+`XonxJc*=_Ls`{)FPIlTJC1A544sR2M)Jisr?^VcA+IoP%wc+FkGNrU`@QMIT7%Ge66T4JK z)>~rB4rV$HN&1d?@klchuOaSvrMWQ>9Lzeye&hQiVBN;``0(S0eTuz7F%U%BD9wa- zeK;eqCdQ1EUq!u@ zg$Llm#Cb|;%LvQm?7~}S)`L?MC1|)t?H9l7vv7E3$$szXh6Se2exDE4bg8I4IkD(J zURzPoxa5S4a8wWuL&KWeyM|qrsj6!>55uy8THHTF4I$Kyi!@MOv;~swSJ&@StHCu4?c^GRbo4jd~4_xKi zHq-CoL-@>9$(U35#$konj_f)S;TbMqd7zcC9TF2ej|AXfm1Jh9!{(Z4D|HzK; z|98Ba*eTxtmKf1`>{jb+%?R&{(#U5%{sHG~XA5Z47o2s2=pOo0X)Mvo>Pl?N1M0CYX&?MpDT z%`R})co;>nOY<`LZYd2hcb2XAb?1Nl)Q)v~0FRO@e8?TbS#=%U_EAYHqbL3{ZmP=U zi#1*=XwmplH6HsYpW5Xtc$dYFDXCYb4)@bEHC#ukr@s^;i^msQ%ysvW&D>aCYu0h) z(9}h#RpP*`ti>i|Mq08rO)m8tY{;JE)Dnw-QQ#&%Sh^THq0;wM{hsnfLlTHtm^Qhf z14v@grRlq<5VUvSTp0P#D2Og)kC7fN+q^~Wv zO~J*cBJk{d@q9iBMgtYP-@P8%k9Z$@)u+XFw1zmD-5;sS5zPT5s@=W=`ftCT9O=>qg5?1yzc`f3EC&5OdJ3f26Ck^Jo|E ztkZQFaFBWq(9+SVkXnRpB4=?d1$ENUK}VOI7cBf0AC(`p4Fe22oB;H}_&YbV`;lh5 zuZtpiA~<8Ir>1Ao6L`Yq?dRj*?JiqbukT9OkBx|?K;PwKFqI;`(JR%m`WzJMDfHz6tpF#galW# zlt7-|+cBjeQ}E(=(V@oTcP5;~2_y>>RA8og^xq+uBq-NuhW=>1;5(lQqV4_H*uO|O zy-%md$0gf_LvPeu<}>Tp%kw@uf3`mY2cGnZ7zLcPpiT57aC$M4*%VheqKYZUkJ5dJ z%rtR?D#MBs!i-|eWp|O~$PEFAiz`IfiL7v{AdT{@8<-?)qm`17++7?29i-qbZ}5eG z2i%_9dI$arA==O&6-gJ31*zYUu+K>J#^Nk6yzm;BYzYp`L z(tjToanO+3Ft|>Fja&eViiO2#tw1Qo1`MiFn20ax+ez^?SYMiwJd_cpVn|a(5Fc$s zTXZ#(#TAJGNOjOjlpN*29w+4zU}JAjczbah=x=BDw|+R5a-bGr@Rp7IO(rI0d&J!*QX|14Is$GH5wY`2QMBMJOr=)8EHj?4b^o}ym|=aqNyl( z&32avIs!)edejj4ffuzh5&(QKh3+r{FU&sZIiF8Z0UAN4NPI z&yH)Z8$46MK~DRgSh*}}06uSgsMTq}l4QMEp?p~e3<8vc23fnM7oM0k$^cmQEE%q> zJY-z_OFh0F`YYXG(o!mbx1)dHNEHr%5UPolEvKH&mIi+hm`o^uTf4q5C6C=4b#zEB zD;ub>L^yC)->kBvtwu-UB;3*FJFLFX(8#_wotgY7W=eW+n;Rm|DL;~C=#{Dj)f)oCos}m9sz@~J^$!dt!qbaEaZkVPP5O*X=&O$~|pl*cDQ_lZ&IV{HT}9mWG#N7glg$P{&Qxvhre~os>cm-F0_L zSGq7)bt~-+*o``5d)1YbEYxiSS4hpkirxiM?5KkP=|2UkhfraiXq_zFtklHx~yi$L$cg#xy+t9uhND;ik@F z!9_0r(&LYO509bPazFxU=ms*^WWcY^&}ljiW*Z2POv?ifVoh(#jc*WiNyQt8OMbX~ z6lg#Kfm50ssW~Q32VHhQc1UlXd^!lNp$FG9i(Ls?7V`9&oDe1;YQ;d}Mm=KnCSqPf zh<*8O2&3^%sugwW#6RJ`QbvtMi6YepBN4MqDsyU!tehTq2W!A7q(PD~F0b#{?Id{i zQd>0!D)-UJ5rfW?`u9Ct`p*hEp?%4nqx=|gn#0`;6Nnc(&-l@wH;dndH5e{RGKX>S9GCpx&lnu*gdJmiO?8gjx6nbq}0BSg>bPQuw{^=hNw+A^nKi}`X{gj|{8nFaR$3gwFt;Bab)hsuUb0gnPZBDFl%&0Xu7Y0DZF|Ap){JV32==atZeCnEgALjY)U+u z@T*4=U@C=F%fzkOEDL@1s-MWE2bAbz{TN5VJ<(4yGb&IO094yLxb)59!AM;ODJ0@r z`N#7R1QU!}6C*a!@KlAXeN3X{6|k1(C-WNL+dc{Hn_9drox*SNL>>bC#%L$v1s7SC zc6ReO&`hfnrBm+6rl7-muo5fu-fgFEQ(%@5*#6ieM$8X!`XVI6nI+}l0fsa*pDJ$r z@O_emH7%=$x+##+1~Tp^=5iB9P~ZkZa6O^T$$!0nPg;{4D;py-^wKwc|B+Hh=aW~3 z@_>Gq&=gUX#Z&thsV_Ms%s`vY9IM{%m<`~8PirHM%%WW&mrt1P4%|14%2v zvjE#rcDz_$coi^d?>V(7LaPHpqacYyM#=^AVmFG`9`K5N@Z-K&e|hVCKq&Wz^Y|P?qY?so;V2D zF&CRFk9F4`|IKjQPM4*}*5vGRk;JoUNPBVX_!> zF58-rK=pJlXKVE)7SZd%9QtLmy?sN^wWEPd%aY6!XO3P=%hTSBq*XE#l5#AM)YL`F&JLsyh77x;&MQU; zfQq34n#TS&ZJL%n2a_O97n5ik$Q_!a-R4LmgwZ|sBY96n!0s!rL@iJ9tG;^W&Do;H zXV@TFZDQI#aErm`sQD$1%iMVNi=itMe!M^uoud^Dicfyf8`m$}QD!jL-B4Ib0tiF| z{L+xQ0Os=YtC~vx%a!S%t3iKhBBW}J4lhl-f&AATu&>&a5US%4278@Z5@UYER&p|M zK}Uzy1Y_YCvLl4gu#eqeYe%|U!FZ2~h?HRAR9}p0=D+^d6$+}&%=+BYO zZW{>4=RFw4MFHX>1NN1@JCLNGN)rF1WGr`AfM=9R}D(y76s&j6P9 zx`cdFIo7Stu9fB}86ag%V5~iGlsIPjTy=0N242u}QlD8Bsr8`DT26Zdq-cZ3C)Ct0 zSI0^lCMCh(*>oHc58N7Lm@0m%>z9!rq3RB}!N}(C|5D}uaaLUX%Qj!2fq*thfPk?6 z=N#?7Rk_KHww^8O208(cKWP{1767xQO1i~^*ir!S|Z$%?mIZ ziL1M+sxu(jo7Lfb$90z5_nE^}`*F5gI`=n=8#g6N1OCeoLEwmEzC6hVLOWiRSGh|< zxktH=R4<`;$)Y<+`{;Q!Y8OYU#@Xvi%)X|WU9wF3m1srVlAmwUZMZrdH;0mX0o;b+ z%P%lkUQi^UB^-KR3^)Kok)kCIj^IKTKPuq+rMhqR!am`vO#3onwrk!JIr;oXI^n(W zhF^^d-9R&v5?|CF)+?IVaiVYdVK00%rC7j=bcsaoUV4-#=_2({23DCi@BOPOECT!Q zg!%$8IF(4S$jE5rmV7HeSimj^T;XqR*2lnFi5Cpheh4hmiud#k*-dJ~pGn#yFc{OB~#sP5x#Yf*A&!GH3dvWoLBNcUCUa|6I;!Ydhtey*< zowl*1*AiOXERSjmgnE6(zD)GwCyUruuZl*L7X%O}^Bw#{%R0XGm+A)Il$9B;hl~-1 zay3BC;{17=^=m4E{ePXqM#~pCn?7NZ&G zbLE$&!3ZhK3AsU(IXz8 zcdTr^9v1t)5S#@Unvn+($(235eFoxF`|cB9CO4Uwnu>hJI1mdWcep6<_x2wLoWOE@ z4PP&kjE%79pGW9;>1&^iIyxgm9JhuWb;KeovW=??LyMm=o2(uuxH<=pLMEP@)xTRC z8M3C&EJp9cL7x(hs0CcXAfA%YjSG#cMgUx{#B7>mGjc`J$ zR1Z5Gm@DwNXk|z*1fm0A4_wHC4qn2)&7bii$Fr=j1#WFV-scn5Bd(vEB1C6f{~!d6 z;qiFw8(zhE_I_)VuA24s^9H+wNF9rrJH;7kT69EC*ZF@HJ*tRE2ZTg5Md>2oed)sS zPBbQcu$PvWe;9FM{GudL@_Zaf|Eo>0;4@?UL@pcSmRXM`qAsfD28^to)mw&*d zpFu$T9JkmYWCQm&l-Hk{op0Qr!yPu7e(@-NFvtt0me26!4ZFgT*xNF^mok{yk%r14n!L;d@edm<&QL0Nv(0w z*Nz<;f&1zW1MztQsuQzDX-i57tFZBcAqAY{y4Dci>o8`%?;GBHX+7Coocb%u`h*`N zxOp&~zATcA%|`)6P(y}%a(V!{sd-k$u*>yiE?MhdcK#BJm&Wk&V6lVUh&10wNMaV1 zE}=v8-n28NPiM`IkG`NzQ}LR&n-#didFB+p4dCqpZbQt2RD?*=prVcYDa>r=z!_a zf4_+FW4UO3^U+bPVOP-_G{HS&-*5Nwk&$0}ca2iS>BqFkWH`inZ%&wl`MusC_9hHv zf}4SL^ys@D^$+Cz`CBY47&M=n7>B-RA$t{6IGn16K2m|Ru0b(r4lu7&;heT1^Tiyd z?BmWXP`8B!LT@h6@CjPT4g0RxT*adM5wMn^VKjv!N#q2cG<7Qo2|Wq9>j~AYjfXVV zhj`}U6<`T#>o|SChOY7@$MO@LxblnLKOGhFu1auK5{B9ea@GVNae?+h5*$QHPuT${ zdB%%$S#{-0wA%G}+1}`i8)po5pq{~GqJd>-$lh}3W4qIF>&|Nqb)eUMaP|06Rc--l z`c$dVHG2}SYUBrkkzF{)sQ2sYc~40zv2%1ZaI`4Dgp`GARaDd>b|#@LjXqRRd~UbX zG5Fv}LdC{eqn5JILZ=n-OEzM@9fs{vmaB8J3YT!tk*9)R2l4RZyNr`RfmdPMYIXQ} z81G5M)=~r`?z?UEy5!P7ks+HhZ<(#$Dnzi`+HS!Zb;Im{AdJ%xijkuj=Nu<4YO|Qy zkEV>x5H#Qz$?4w7^PrO-MdB|WOSU&-ozS@UH!d>GjHB;feT)Dt!q1P=gZO6s^!{hW zJs!9mt?y1e119lEoWS?Pi^M!>cT)4epSqv-6Rzvo*zMyHamXWuxcBP|H9?{L++L&D zoRiaGZwY1@+QPn!m4?9D@kzsspZL%Nwqb=JB{aPT;z z3w>j{V+X7`2MEl~D$)w;+Yg+)XAf-$-=)&Q{7vuA)2eF)yZGD_ZYh$>E+*5s!OHwt zBvKY+Mi+@k1@U5ZhoagMJv(67_rF^0ZR5O#sP{l)A!rNpew5M5%HHl_t;fW0x**_j z{=vH#4OS64a1s{88b)^ZMd0u_1Yv|jDLXQ8b3-o7quI;yGk)W5$u$L6=%*6n&lJ_- zxBPne+n7IzqqPoEh8F|!q}s$Iu1|*4Qqp`Z#$Dg&z3`~4V6Ll^k%%?3Z&0jK23>Lb?vksGye z9}WUqr%wufJ3Ump_B4GNQ~6@!=vWp7}D0}=KP>6 z*^UsaL#6KHBtr)T4U>L^_4i5xD7As1Ha{x~i_&_!Cj0A@Rgim{7c@RFL@`hjsgnxx zyVK#D9!Tz%%C!&nk>c%B|I8+i=YHWtWg0<2d>f;Or#K(I4@*dkM9= z#Jhd8UvWT80ehuAz!G3>YhBGZ(d$}|6Pp^g(0?a6Z)%{gDK%oQE(7;^aBA`Odl9-A zu*~B4>RVrTv-iIZ+#FO51qc3cWP*P_{3Amyk#Lp7W#z3$xatDRO}ViOH{app0_`#iRss!(riZ&w;zEVAwCxn) zg!1iI+|7FJ`T`flgw$;7FRIsgE|6mW4+h=0ZUZ2p4GnZvEeLb3PH(UOa|>fR-VY2e zPGnEMy~N{kf1L>ai6B#92}v-qvZs+zUxQHwsE#iTN&Jn>j`}^re)5y_;n$F6?kRe@ zW#@zg?Cg|v%DR<5V`&ZHz?;Kr$x$Y^c*kxf9>r$49F2P~prk@3P^6VxtmnkaEjtOt zNLN`(^8<&~y2yVqtW*3Oqc~4RF!8uJ709)7BflqzRByj`$%ewQO=u#tSkuH548<_` zOE+IqBMK|uB;anKE)81ZFo-E8uch#O2A|vIU@(-@d|KS9)vJ64_(?<|kF&Mj!~B4V zaXKses?{ntIfj2vf%*J$5+l;PUx7p)n11U4Jjt9?e}-}yo$Aq>SOx~kJ@xpXYl&om zR?nK;J)V^6X9O+d1nJ8TEU4?4E<2Y;Off5)OmRAw&8Y&8Hur1h$K{!Y$J62QHN1gJfqAf&Z0h%Lc3eh*5w8Iat^|zTxiOXCZ^$gf)L1}bLD_<3 zS(07QErBaVV6(jF4xl7EaIh$3NBf-qmX4THY{!G%Cie%mi*p& zD6iuukXNSn(UVhK`^)d4Ns?RD@9vc+pUY9cvxh2KmI2^i=$F|1L>VuHmt?po7sVaX zHZJ%8WYL3Z;Yd;U2s2+j>ct?}IRd2{+q@;Mer40Vv&5-@FEAP<(R^VRFlRM5l_h0@ zRuD7F$SGe1k%G~@TO!7H@%jh6o-QgW$lNi}{6$?OHD}Ckoee*$MtQfD08=at)xc72 zGs;N!ii^%c`Z(zw7_l+p$yG27(>52jzDdzTI0cx7QV~2^K@Q)*<8WK0h2qFfzRa)H z;ZJ0;1%`EaS!n=eMV)Y;`eC-)5dvKswX+K6i&6O>Elf1#%aIZ7f6JyQ&&`2fWWv+S zV$wYOR;1Y}tlaM-tf7kXmbfqVI@G|~29YarX8!bh?cN!MU&hKe?aFtiu+Y_yajh>4 z87Fytf{HF%g7&0~|(!m~}xGFXs2q&uC5L52`be$Qc?!~c!iwA!WxbxGyTDRf%XyvS@9`?!k=6?L%RS6X4OgX5IX zXNk6Q1i2==nWxV<4yBc0K?-MEv{d`6E`j3-8`B)UZ|}Lh5@2Gm8~_RV>K$7FZ4qma zD_#~;#aw&hsBimX3GcUQl6kyHM5P!KI6GiC*co?(jjCS~p7Nu9q+OYYMZ}5NZ9a=1 zL?&;*u514C1t}?+KWABo4@weHD=l5Dnh9elNp0Jrz{&)en?yGokFu@W1qlSrD{N9M z)JjrAh&R)krD^{;btO#~ZD$)xq&X58>@J$NiMC3QOJRl-pK%I}G1DG~Dgy&z?(~TK z?(ePB;Al74uoE*J=ke??rm7XP3q~xfk;4s`MOq9ai!6CDE8k5Jsq!NG7@oy$G`&W; zJ(E`aorN^O9R%c;&Ax~k+OZ~KMy?ANO)-aJu3~y;_$MEJURbLxMA5|G)1B+5A92!+ z1n@Q>2Ua3vzge7GSMtr?K4{s%3a~Y|#`{8q@D9u7^ZYu-!}fVR*k6gSe;azcL)8C! z*%iJ9!Jxm<<7`i7#ZE;j{s{Z0FTJjMSj;5$8|!PVC7XY*bZ3WW@A~Uy_O<78eO)z| zl#W90)LT`Gx*LgI-!d+9M@*wSj$Y8NQ%+}uHT($N_KJlr3}Xy9+Ncl|B_*%_nkE7X zN0=m=oK|us?Nrh#&41|>Pe`UIsnd)+aer?Gt&j{%v(**IPT*=a+k(8q>JvK-z0EQy z+=p&7aY`~7p4bV5J2BX+pYpHX#46aH>7m5?B(kmXd>>spz5XWbowHuRfd6uOk}49t z0Wlf9B_DxW*$;Zlx|4tkb`*(P>88ZTPh9`6^b_j`wlKeHU@qnga4u!+;FI%W%{V98 z^Z3TNfI!2MX~A>-d+=0rkNZpSkS(1CTh|M4)V;*S_c(d%(RjuyiY*8_A{O?j;6~_(82Ayw=Bnjw>eA(u~M;(a!bA!SC{E#pj*SSg);Tf zKSr*C+BQb5syV8Hn|8%=4NDVKPE9F8oQegVTsJGy;f4!#{i9eMJf|oSUa@rBWD)To zV;Fd#ku=>78K7juzEn_~<_vNh74*}>h~^Z-C#{W2pTW)UJRLoU0m|2M+zsn%R5vT1 zJ1>8@cTl!6nHtYxJH+F#gRp@nG=o#y8+t(&Ax}boHa(>eV>zvwfRM^*0ea3huNl{d z)nHty2n5QAun`-4XfRF~hQTAF^FW!+^-5suR5VV{HB(_JLaeC^DQSjgFZt-FK(EmW4dV_h(?(@YA6 zciLHP-x4PpLCR0-T(*xO)gvJ+r_+Ck`UZ3|)O>3Ax| zQx}t-Ff=!K24fkv9R#aWTy|}hgF9w!ZIkAvJ(;F0%3wt>vTIrqv(0etnYp>@dqQX( zJK%ASar?PfdmG#e5N&(AA9r)Dci&cL@rV$Fz3e`?J?lA;cL=q+W9x(WMd88W9B#Yb zfgHZihpzWJ;r4nve)lW4YwnPH-S;~~dz!;5h}R;j8Zm|JQ+Hv+NDHisR32DEc*!&TH^m3!5(rp2rK-{0p?`7mA+Rnw)_4Lh4{MW^3$fzp*iON6c2)InGVC~NBFyyKlH z*5X{A?79>kH)EQmE?;@LH|4Am$aHe)t6#h)uK+!gC8bbf<27%u!(21ev~aO_C5i#-Wi3zt(r@bbD6C+PGgPxbs;ICbjA#J$oPCy6nfMhfoJ zviMp_;8TO!u_5wV7`BV7WhD-j*Pd1txyw5xW-?dBtyK6=YB4P_hx+kuzs;WJB}edR z$+3gg`iiLU^M@_-I8#^J@&|;jlp(K->HW#{bO+_fmFhq913ehmm<1}o8;cwtDs5U= z>tQs5mBCH?EM<{`8#TRPf*XP&dxTk92W?hEDi#L_Jn;xwG?p7}!myK%#@h$I!2XIY znjBYoeNp#9M!G5`t%aV{BCB-%ZKNe_Ou@2Kt=3u4YK5bP95VOp8zM7a8T*#o%TEe6 z*D=84o**|uUD?D)%Wt2J8Y>S9dSDf5b3-dg_N+<=Q?u^>wL*dp%*e=B&uvu^IG7Qy z3|dvg^!1l<1KG%}QT7=5&;9DyzHqQckM~(063IUr*y1?DGA2{CCOVS*ka=OJu?@hHL+;RxoIoE@X*5LxatC4wmkdbg`pKxeV)+;EwKUU z4SbcpjI$ea;xt)nZlkBiL<0$9mu3~8U8DsDo`txH)^1)Vqf4)}v^E#opyMR5VM|mB zdB()l>v&=4oNb>dLMfhP=w2H+i|PAc8lJj6{O&LMeF;R1G`s#0^O#ai51>f0NCYrf?`g64!1@ro z34?L_c0IpgcYhN>0m8si$FZC-ZiT3z<%ixMhQu0*y&7o06t)CA^UyVq*2W%rJ$^7G zdVKHPkP=$o4s+V}2(JolNq0}i&XqaOi3LW9Xy!!9Pmk)2D$}fISEg)&O-OTy6|ic( zdB2`&4qz=E+U|P(oChno_4&GcoJ~pG8SH*vPwr282VdDE=zp!;Pp!G#^;+R7m{&En zcS%o4RJNwN(ozWLt<6hW=GZfpYT1^L_T?H7lzdH7i3Hhs4Ul$)o z6uDlb#yTpE9qNN?J~TM}>;N0mh$-G9jh8I{2Vgjm$Ep%|?3)ANaNe8xZA2|ij=?~a z?1WTwirg`$l4e2_(X(}OPzeWIn#-|~3sx8gQ)}Bc@RUv4s-s-Kwrv#%;sS0zLiirw z`UjY&;je}~en%(A0cET}UH50`iTqWxIPB%)%x0+>$|!tt=C+N$(q0YpDa@GZ@w2q6e zWCv|>D|VY}FOG~PUA(13P6^W}g zS!?WI=u5kYuH@SKvWh?WD>oBhM~2;1L@iOC+QVa(G~0ja@^nw>kfP@3-J~qUODPqdZ|V z07Ra~QDF1u?1WqVAh*Te4d!4VAk=Kin5rz7rlbot&r z4Ww3mQH=Hid}d+p?Dzbw-fn{jV%y#|n+kS&eC+OD@_?-0Cr@r5^8kBY{OymsuWf{I zfId`x3OQWkrxP8d-PD6I26%xl>wjDoTxSo@|IQ`@gH+ z=h_!00e>)zF@Bc@()>pt2(_*!w`O(HdW+Gdy)PXaw_q3n@rbO0gCVY_AHqa-{piUQ zhe^jH`KiSpm3@d|2zoBW0uAL>x857jTXy*Qau|l+B|mofqoy?cSaX$;Hi1%GpbLTK zZI!#fiiR)y#=QIfqF~i}Gz^}>IO!{YDYWQqRYN{UPCNsnp2E6=`h}fqn>&|CM$fi> z@R;Pasf;$woN|>&9XOM%)7fr&gSN3VF#osS1xa8U(RMNQFSSu0z8zZr*L4&khd>s> zpr`TR9dc(e-ooI@^9OcH)QkNrJ%Ct?deO zFJ1+*@=xx>QwbU5m{}yXaUT7nc995O}s!c%5s4L$q4WDSc zC`XNIwHtduL#270eirlVZy74Xw>EKP2z?r4mq7##M_$nmHq0UEuwByFZLqo*ti z_*~kjK?L}P8GpWx?9A1|?fn7SzPj;FAE>%DyRgP^@AZDy0RoA~b;jcV0F7|>_yp!$ zYhCnpLk-tGxY*wI-T?+*QkugFYNK-4bhDoje+ zv-=)(X@}OfQ8}@$od!|$mT6{N%N}=aY}rU`TSam!^We{Uy;Ven3P4EXB9v7p{$3ek zUEGHDIYaH$`V1kA#RL89Ycy(b1LqF7^Sc=q4qjwz3c__sc&x5sa4+@Ep*foWY`&Yd zGP7%`$sQt;K{Fq{Bv?uRCdvV81N~ z^R;TQA-d1jxR{`84avojotfyP$!QS-#W@S=Ga3L5eemP~%C}v-Rfs)ue2qX*dV^lR zY4R!Xi;>xi)-Y^6cM`DyTIKqt;d_zydT(hn_-e9|p)GMLx*<14j}OnUZ;iXqF;a4K z#%8FkZ_sd)P(Szt4_CnJtm+_#!Kfl#dAH~3VK%1?vhO3TWJ!1LUE zOWCnhSH3&*FM z-rJFV+%)=~bc(#(v==}%q`c)yhyWD^_tRj4Z*}=(P4QZQTYr5$gSEFbVFrub%U1Pz zMCEoUmrDrj$8h5)QO-Em>MHJG^%3wp$+zFQLKp%67l|<%*wt4z1pCh$e+0?_w+~D( z;MFpC=fZe{F#9Sx{TGb}<)OP`_N*Gn1B<%SU7((6PD<+>ynt4cf4BZG?TKy@r*H+n z{i+)mGQip!o8)YMzk;e*H{c%Yhy<)J0aGe`r+BCg`pC_@_<0CV;~KWI1lDo&K!CTol*6RHu#8dbjKyv%i7^==%(+>&Eafwe?%u3*S(Fno5BDh@O8Xzh^BEa`<;VgVfnnd z^f)G_n$hVIO~~&B<6$*V>p!0J*pS?mWlB|6mIX`Y>blz4sCPD~n<)OvJ)L@e_d7t0 zC={Bs!t$!Brx`W9a@eQ5|108K3p#)PJErX#xje~A&1CPNJMj`-; z1AIJLaZE^O`)UY&YfQbzLUp@!x`0!yyo6>|o93H`NnN6III6V%sz?S|$(=TJS|Sx2 z(S+5{{Z6)94irxs*gNU5yY3h>pJ!G^d7aU?Q)*XA`K6l_TXnOzBqhjdfl(YpyIl{la(4{I#Wd|1nRyLrrdKa;L<_?!Y*q`Y#U@&*&6nT28f)_r58 zg|RB@5i@0`lJ$9gr=gp_i7GtnTtYvpfmu;|&<=G8sN`K&vM%|;iB#d{?XW7Z2JsG18Z zJt$=6IrP43sdZ8YR%0i+7v7K_zJ(9Ih@jU{zpL}&v#V+g@h^A#$KdQ;-S4_X!|dyU zvaLNISpD7mle%#Go!yTU0|LDsexIx89iBFKf-e|rNDloI{e%7J^e{pn8262wnK4mByHgWT`RBd=UZQC_M`(i27H_D9x@#)%( zP2F){I|iyW5ct3 zhGd*D4L=ZL&Q()9$&*2Ah#S)1`fE&DUqiaY+%ZqYda}j<#^gks?51rKxPP_)>5exI z6pwR2>k3jJvN6-l-fGgX%K3G2`w=fQANMNp!LKZSF5%o}+l~MJOvqsMG%{7l7V5Q^1A1Lxz?r4u= zsQA3RXiEsECgDTd_4FcjB}UWY)#P;+`7%VbX&|dRa$V)5pit2##;CUeC24^v=*&TX zwoh(~x0;EGy~%>8H1Q0r&8Rdgl@)A&^eEpYp?tg&nU_cNOC+~axZ?|zV+GwnMj91E zi3To9?-;02c4)SJ)}k}+`B~nj{rFTnli(}i$=Ir>ua4rFUHIR)xwR7c%VWgM${Q1; zQ`&zz;Bl|SeiNOAvWsN=8vEg9qbz}8-R9}eVW-(W=V_?t0h%x(EU0djH zt4zKFDaFjX#iQJA^zLR;y2I2@X z^uk#BUyU0PjG9YQON_6agbbx|dVi|j-L1!oWbYhmQ=JBgxwL07(`7#3)e$4e3k&Aa zN0-LNW@d=o8vHx+eLceD8nOCIP;v_H0EZhn!-{3b=Q3E1suYCSsDt(jm(}}W&zwS* z{hMj=&%>>=`!B!sW(Ngq-KJ0+)Gsu6Xp0NN{=?6VspkZHlhgWOwajGtN*2zn0aE@) zYfGyB(YXxE<@v(Ckwn^o=fr1p`L!Q=>bO3yUHzS>d8!P)w9zW(*r(RW7Uk|^;ykZg zn*_gv*>wi&cO%=hWjh>(0EhFjMs=}G-HZm?=mdYFSy{CVk1L%AZeS6=?0 z4DR#*hJt|v0D)4G1ON%~!C`a|HO+*q+1p-jj6ojPTiknrnJG&DR<8uykpwA6<39?? z>f`X_kvUWujmgTS4TqB1__fNFSNstV2*YD3$tn(&2NQ7=r=K04GgCmx$wWty=*;Nb zK5}k(d_Esf9?o(Ha$!uzao&f+s3t#&4?dbb`+-%-^!R^Nk~Je2V>KPEH&mo*WyqHl z11;245-T^#Qa_S|yoFR0D>v#=KU{S6!Iu{NYtmGuEV5?GOQL1qD@z2kW-3bvOOq;@ zsw;^tEksoQEH9MXHpr>8R$QcOVlBQWS4LM|1Z!r_z5A+UC@)jCu#{9TT3TqTm@O?t zSDq{>$_m4jPq;qwC|anj7gRP`TBxeHRhHyTK)P0xY)_zXJxt&$*FiGAQ1DL<>7rgK z0qwz8tOjFz+PJ2#15a4c888XWw`20F=l zpYT`?(KmD5XzE7k2d3$9Vzy!Wu(~nFhOhbym&#qFW`bt=;994Iuf*!ztmC-y z1e2Yk0_aZ)zLFcLIXcuJzf+;o#BNNY8t&uDY}MAnx$McfKqe<_d zE51_#XH;%>rXs8C#&di3fM^N+_-+d4pG1W`9#!g^#JVO*59LU=KvRxWB7WFAO-d@3 zOu;1@$;MVybCj%MON^K`H){5v6s;J~3bb=Eup@`-fp<@vmgZ6^rU!Et(EPnL)k4mC zr)C*E>gjw8&=5@oRLK@GYN(!}hBl-GKO#Dn=Y{pb!>xgmZ-pBk)g6#aYLzrBZYo+1p*Sg> zGof|$3)(;oW2=J@hgK*>S;Zo2 zH^;@4^VG$((C4Ig&2r)ijstUUDV-Ve*TGb;PK?dC%iaO+fJb#w7hDrP)rBlpq{}*L z-Q8L?Wc_W&g3Gc(*@QVMvDwpnEfk99hk4eo6_-q)3u9HuJ*%hwZz%{oU7D33@3Yn? zR?if@ns&=y9u_(k+{2b7{%%t>tpoY;*0mwVQDkbnqZgw5<}W_71b*~K=(uZZ5+)Z6^lkVPw)Owf86uC zNQflFBTnM9WYuN}$&aPEFwF*JIE`lL$j?W%QX03OQ{V@56c_eum(Xr5Zue{Tg4M6= zS5exjYbG%5)0%fDm4mjWmtT|8D{)q=jH%!cU&sX4tZ%tOab5{Tx^eK*aqPl!1nkO8 zu$qsq#pdx?7Px2eUf}HT@jhwgey&cC%6XoBoj4=fgZuit$*#58p0!&kO+zO3eZ`HX zjSJ0)p1-y))TYT(reb_s$U2LPjc1NqrfQaUl`UhYQ$88+(RFc@vgoYZ%I_`IYI+GC zla9XhYsQw6g=SZsN$)tX;)4C_N#)WFgSF|1y}V{qRw_>JX|-?a))0S z@cD#(wCqwc1M<)jcLL95t=g#7@@AdzC(x#D_lC&EkHf)jC*=Elvuko z>{PzKn1)tyj8omFCUBfLhQHHc=F=(**y9=((XJ%BOW9?rap`covlGRj?s1nn?P=3e z%Bznmw7hptm_6$+MK0scenp!(OjaZ&#yN=}?9^4K+L}f=_s@9Q)MGgMOf9ZWn*&kM zyfz+o>&A=@)5cUJotCkUL-<-U29%-P{848DcsI8SOG}(J+YB|^EH|u`C?m`HQaV%Q zrwMJfOc+8?AoVcHpXv3Jf|{j-x>{JP5|m`FJAJgrFNG!fqI9?sQ>WLx!+lDckOS}s zx_q6QAi&H|d>nxUf&~iX)zb~+-WW1&$OBA3M0!s6yODWq*%@oE<^0#1e}IWpFRZNM zqr`i)tq0@@g{PmjGBhjaner8N*-eFL6h!Hn803qLFiaU-9DrKzFyKqLd6X;F=5$kW zlATVn74{1q+(3QcoS`f5zmkQZ7-B;_YN1EYRV;G5Zo}yjIF&A$_U_{t%$(dNg(womeV}68!4v*QZz1LX(0M$CRi=V&LQZ{LqQrcc zJ_H`2lX~QA?tDU4h8ZR_iEeDsgc4Z>YW6JO4I4i_D(FO>jzgjF>|id}z+)0FvwhzLQ?m09n`>{)XquTS8c^frpeJphr%MIKp!4lIj ztfN=% zdM*)uXtK}qFAG1NC~fV2me=yNglzfLYi%?i5Uyk$=Da3sJ{z)|V<)Ap^t?)#WTR!C zO3ak6MGvG&&C4V1QK!RreWi=8;C?ogGTM;58&_PZ9_pNN^b{%6^jefvRP7!Kvi(Ln4w~$Qm9y@p$9pD zphd`9C{|U~9fjR5NsR=}XRsLnZYWb88L>zPZV5@$vGPxcI-^CI>RnksS1crrY1f!f zFwc5*nbjkRi;)YObG6aFq*%WWW<8*DD8d?0J+)%)k>c0w1$iDW-;RuY&>1a8%U|nb zn3p07zTDHU>%qS4Zwxa3ExO{cUv{-NxHI|o%Hm}6Pi20izE#=%&>;h+tx>Mw? z?GgVFPSTq5^Vr?e0f z*lB-QkEjT~a=ID`Q6X3H*y09QBUrR_KMR8>AXGe`1(HR&1EvXCVmAIbtn-X%;6B=( zhiJ*sJoas(kA@`^PpmL!6k5~AuYhioN<^TZW_wJDMn!YBs(4Iw% z9XW{SApzrV59lNAkT*EvZq*8NgeY#wKOWsZ4Ur01#U|unqR{Opl58lTNUk(c0%O$J zZ2(eMCZJnoj2nx}4%WNK#NkpOm4@dbUK5zj~N& z1As0z-zCM~Xsh((=_AZQAr=EItx8lm7B?&>wax{mX^w(SSUJ!goH!>-en zVbv@gLttvZ=L`y5(PyNZJVG1c(q73|G!aRa zKEV;AbAU)?$;x5}&5+F~GiyVKV!t+cvhSTsO|yh$#Ayl(0z|&;TCNv7tubmx3FBG( z>kWfV4sd=O^e$}t^t9EcEsj)-*vvH+qYfcJ8620Dt>n#^j)uVKl`t}Z|%1}pIN`pe|zq4_`WV&o!cFIvyfGU5M4X-e*Sfl1Z%NhO**Hh1gU-w|L0NG@__BP70AOpx1Hnw1k|UjYAzq=LoT*GuC(qrG#?e*oEBw?dOmr|&-aH| zS`_u=4m9z`xK8ZWDFj?cyZg#eWAT5@MaD=7VYk6qJ8d8(Muh5@q7mfLhwICZM~_}4 zz#Sei{xU3=ftPU3({}-gX;i3T%a!0F$4FCa;*&@+`u zXqSR6aIHahxFpLh^M(#dKj~na^)jZP8Hh2tyoiXD0Swlq+hoxJ>vjxn9*e0PJMHJGwcC#7H}!_o4~yhQxtm=03Y} zz_dUrkvI2gLrC)$#D1{PJdw@n{hM9F;0p6M=5UFNr^a7&8`gyR;x!ZX0WXw79ftH} zr};8j6X;vnKr|TQU;kEbRk5AZpN%9~J2&B3?#qLgS^B-nBw#+D;9etGv5u>;3{G4r zMj`StXUOWR|9KP|nLvh3jBWCNQ_+9^d;gZHL&P-c{e15F5Tgk-JIn52f1hlvB7Ea&N*^8t4PTJ$pC}>5QbcFCf=!Unq&_a*Xf13 zX{MZ;`mWVj&QSb8ctLU{!||m)g^*o_+LK6SL@6_Pv) zBk`rIg#KGvO3HdQB#6prVv1kQ?=w5vRfCsS@9i%o$BqPb5wy*MltSrXg^b@a8SHW6 zi|?Essj1bH8ID3$v&Q7pNYpl+>B$HpuIyHoF=bc9-i)QLsL;-Muw-F8qRdNW;%wpB ztRY$yE0t9bPw@l`fCCo99;2WHT6G}!xv-DDdtH&@up&_vxF`8{5S`X zqp|&F$+P{nWab`)_SlkW@mQCD51-9$qo_uzG#Xu4?lDIHZJ_})&9mbV+aD$UI5G_zjQDHykPEC zzGrX5uHu8vhqk04rQ2*RE>GjN^Kt!(Y(;Vx^Ksj5Pr_Pa{A2TWoo-J>_j3oKh71jK zx8Q;E@S5=rv_*QqwTZJtIKY@znr#;$AXBn7*@r1&W)NuFEYRej0HykYQP;F9){o{w zFb!sT$WfA0feyu`@j@B#wuj}W_dcn^W%_C|{GZxW!tD=qSV))TNKd*)W6_)MS8CX^ z=O1!KJ!=m_RkfL&=g%SMo!dPb3-7*JvsTb88S2q5`GD&DPY2-|WMsHPwVbHeA*7+h zz+Tn#glIj^_}4`WTm{!Fye!{l?^_a&Ua2QHbSR;=E` z3t$_y0UaP|&UWR*{+vH2is2XX8|bNd@-L32^YuOPEP`k7#VXYp?tci)eRJqNzZ?#9 zd>RB}YrXKL!yzpS2d6WqV|jyq7^wJyfU%$l%;foBNAqYCge{MdH(PfAt&?nU!aP-T zk%rnlQBcfjC=EFLG1!~{D)8p2`0BRwd@O*|ybcRr=_mwokM3&-<}T$cbBA%y^AuLU zNvFyyw89R@FJj@J5}YDd;Dd!Qw9#umU4~&ArKylBVX1gFkC}Rx&MPIs0sZvYtERHy zfr4@oBdUvYLgy%wq>FZnR*y6`E@Q)2<#)AbzTd;U8KWu7y1C3dPns?4&8w~I-ahme zc?VbbZ0l4uJqq3+Y55Ln{0JV$7q#q1vCdVVKfHr(mKzX^LgW$aD!zmCd|?P!)E*D3 zVc;k(by5d|{_cssWhmQ2+0>A?>HkU26xc1>=VzMqeT=ruQ#dAHJ~@O5D_lm?jLAK~ z;wvPz)|F|-H?wJ~|2rAD(c`gl)4Y5A`Y*lvaUJ)W&P z>$0TJZxx@_glZOgci5b%aCR0X!RyM2K~F1@d{hpd~#SNI{;yeR_y4L+yAk%5-`jMH3gU-~Vl@{8uko5Q2Uya=o3 zq1ccsDTpyXqwJ8rqt?mcs`OTQC!eRq(A%fb+}8vLSdY`n>jEzoKInLs;k83%FW2)C zNjH68uz2y&H$3FCxtj-1s%UeQ4)*9Oh${UHi;`tK zfcxr)?Z!|XgVrDo1wj2yIF)YnRs`G+pQ>f$adIwvi1^D>)a`*~V>Xzh75RK#U^u-x z8U_Lc&M(~-^A&S6h`+c2^d0|8RO~0N^D8kK7WBII`J9kXw<7H0M8ct{n7+{* z%vJ?ju&Bq8xtO4Dyi>mjv3l+ihBF?%C?D~_TRB{K7k#i?xkRsn4!Tsm!aC%@B`Ymz zy3*9aJzMe_%V;`^W;x&NQKWGv{=_%~Gp>Z!5kVZ?c|gVC!l1-GY!v8FJMB{m&HCp! zi(|5+s(%`5DRMHUFWYyt?YtPmfra-lLb_ay+U;RQ;$1TSG^9j8)s+H>kU7kxxSK^! z<=-yRwS>0>W*YP?u7{t@zwvu+Q7#U3M|6bamQsygHlS6yP~U*W%_39Jdo9g^n?|Hm zcgj*68qbqxKTZ-~aKBY}YN-whFFfVkohl{r$oD)p*i#@ywcFmNV$5JoB7Ff3a@J{>sGI>`1NxCwkT=d@LmuH!{saky!$7uYWox?k&wnpbm~ds~oD22x)-ow{vM z98AcH^VB;No<|5S3+K#Pp94vJt~8hMrho*y^pschen@$IH89E9ybsdoX)VHPUrjb^ zY3a0k-oBjF=t)YqUwt|M*gH(+W?Y7Ln zyTM$aEdG4Uh=!P`||15LXJ|%Tm{yy{Tfk@zDI|l)(O?J$0h>kn5?^VXpRypOuywH z%^_kBqN`>-Te6d6F~6thIg5tNIfW(sMWyT*>*+|&lmYyM(PFxYjBCC$h*ZR^)H?5# zwK4yBk^9m&2YQXdP9*4cS8$fh29ggw7|T`+PvfJN5mQT?0@E+%r5Zzd1h&+^1qr96 zOJ}~X(xzh_io1=>BLuJrK@6mre7;XTlQQEfaPWS4Vab#GwDWWDCgrxDPs@^0=Jm+i zG>Od_y#X`gsQwqIv_|cTl0sX0esB+_V=6X(nMxNJYw4eW^fSej%cyP?>1ZFa0v=V4mHGp}iu zlf>muwu0Hn`Y~L4t21B2;5za3Q-&XK1fhioQvloChie7n{!C|C&f?6^Q5*9)o|I`$ zb(CE!JC|!)(Tq91z5_nhA)Lf|G4teOk5h!RrMA70KEN?JveHa)b2n`Q;+oP-TU6gb zypuRVY0h@wBBI$^4(b^r`*NC#E5t|>5F?qo9A{BPv%tuH&`~lUd4xsmEyEABDCqJK zcya{xTi02s+{`{7k46qO|y+w;QX8L)#ii*q~9&#j08kjY&-EyoBddM&q%19cebCx zV-Vr5MZESbt2cGn6fcug+jyg-Y;_Knn=Y=V&^HK*q)(kn=~h8pkXl2v^RiHpFqJ08 zE9EM=W}x}bC$LHE!)3cT=L=S-)vIr1-j1KrNmtm;_bi|I2OMP)mX@?!=Sy{U)G_-7 z83lwnq0r-K@Z1|cnYA zA@abazD~_K!Vcjx*;ia=d;LN<2rKaiRnwdmhF5s9HyZ!xT-8-y6%94?X~zd*Bl4H8 zt_xh!2M_1PV?22-@m$1Yk#p}q8Q#Y7A1(eQ;MJaOzO<>4$CNIvDa zV2Ef6bH+i%lwsnwsH~P}ymd6in&aBxD?hl;bP#GlTwx{~a|j->_-s1$;kpzdx3`vS zdKFhfxO$Imfg=-TL7eCn%})m8=ZKNn@PMhGK`ph=8vKyvYK-yc74T!%%Tjs+ZL$f% z^mu|NXxx960fDD`TtK(6D~a=OeG<5dXyCPw`5Zn-u2LXTEc#EY6_Io2zFOXljcfv& zhg1P&P#oc$@~Vh5an~YV%cgYyilW#-mTh$28y65J<@$6_tYC* z$v>Q{mAld{{HB*Odd|m0Py*{t;NY-X9Ui;`I=(a44JMU$m$^@!Fcfy0gSgBKNAZmp z*UjuKA}D#XgNVnkETeAe&7mCkhM=DuxKExJGSPkot?+pp$8Ywi(`ff9^_t!K&0vlK zjk65UZE;tHn_P*V-uk_W?QPWdD_p!0HHDY#S;;H5-Io(rJ$e2I|6BjxPO?Y0xtyI% zgKtp7&Rt53s(g__@)wrSHu+zgYw(ZHnE}@H(E{OOJi`oyn4#6ke)b|M)uLa*&J9Arl{d)l0 zK$}A$?EFs&E)M{)r8-~Ftcqyz(WBuA@iwVcfNPSUI8Pu{YXo~m+jgLU82#KqyJL?x z+vjF*N|y$^C9lznE@HXgXJmK`8MoJi(Zy(C0*u5P+X7JBK5j!9MaL9MbZ*GuGGByI+aB55z# zBzy$Mbklh>!YrLOsLa!B8B_sN2s)^`Xv-S8^e^-fxb~9LF!_Yub{Tz&8JvBP-r^`) z7g1H7X19Qs@Q9%dJ z)<2SF4ACVyFC4#k|EbftikQI@Zk0*_AIKbgk`RDTU(CH7`!HtP?k27cKAh=vd)( z&%1Q0l3F*yQ;U#y=zrKhrfaXCKwWzKZhPQP+G_&z1o>VU4diqVa`MIzdOYx5&VIc* z{HmgEo^bp3nMd>5JaOX?;%0b3K5$yUc+dvnou#yuRfF<9U)qdt#eYRJY}E17!d={M zAV3qwng2Bp(?bv$RphH;ODT>#HjCiwV7ck4KUKqv=xpI*44#5xloQe92X4krZlc@ zy7ZZ?U2sVWv;6fN0q*@dHlXx0V*GR8vyh-}eHS=?t}c5Ahiv%t(Swl27c|{fywu37 zq(=u%ji@maB(kiQJOxj(vjPU>SJf}u3r9HoRiN^?>oLb8{h~QHc}DpYN}Nu9dFEF% zQ*N}Vp&*p7dub|yIczsxoOFH^{$&Eb0KA^iFg5lzHYd)OGY%*FE&eTT(&VgchJoX;QlJjk)9-%N4QiY{Fom0xtSnblwv{&G_SMu^Q0X7NcJMy z;JHJV&H`hH{sdgt1B}B1z2>~fsAMMe>ruu`NWoCkbc2q80`3i<=r{>~ zCMZJ%7a$@Vh)LhdJ_erbMzc;8(=PkUn+T7;=8F;4GFuQomrNlz%t}0R%B3 zA=F1#0cSXfAl-YX3=hZw#ngiJ#F?Q$wgmDh!<}b*;$H;$g383xg!$6oqI#icn!3Z_ z>uPJY^|&reo$ndwK-LH2;PLTsa&WHv2NjPskc>tUhI^+VLn>zM;`0Ld1Ngz|jeem@ zP~M><4Wi2icBv&f=B^1+#l9xWFyw^>I7lC}*~h55gE%jN1k&>I?|&e_s?fz^KuI1Ud@`mixvlikezviXxt}Kn8)3a8{IMr3!4ZEBNu8bBGFzj@Tdi`))A7uZ`;Mnc zO>uoUEG|}5{qdYx5cxyvP{={PZ;mp=hu{pA9|n0yG<)Uzg9Gl}$c_X_{0r$2%o9fv z?3FuhkHblEpBBGI;!JRu7*13A*9{J+^@MV8Q7O}?g#ITu^b!)5B_(o1JOIFG7~V)Z zV4(hkl?f!hcVJH&giul#2N+$RTq(Z~wRh{3Qo?$BE`yYHDBtCVK&sMHaW&J~06Yz| zm%UORs13>v`hg~Pkvc&};ffvuieUCM-Zvc_U0f`@9D#!9NK>CfBz@GUqpz!QG6a0H zaz_g;@d?`#u^{@j@}dafZYWLzT96EHxL)1^MitXKKYdU&ff$)t6R{eV<4Kh-1+yRE zghFP@0<9LJLQ=peQAz#ESYv?&0kr1{xqOvWwk@EEWEBrpWy1yx4MN<-{tuC7^kMZV zp)it}h5bA+l|mOyS%~sqd4NGI6bRmn6a;@Q)aC;|JBxE>>aVh zRu0Qz$-+t)nxL303fGpSm10Iks%-rZkTue(urk3}T%wk(iTU&wR(w_v4E`jd> z3{`b3>28{InjcZUJ&u6$EE%$GPC+eW@F^f8gNIjJ5qbJN2oAAW3^~Z5{%sN+y$yY5 zGHRU*rNfOShs?W?ZQh7d;#bb7d+uc_Dl<%)JAsf&p(KK{5vn{Fm9i|ih1noIUT`|s z$U;Hx>Yp)Qz{SFxv{0iUM8Os8>dvbD2&8aAUqZjlyWE?g$5O4@KuWS*R`r_50sZK8 zwZ80Zf9X-_+Ey4GU_)u(!HHmEn{IV&iHssz^aJAGKbx_u>_Sg1p6|gG&6V>?jv}*? z??zAn7!I0b2)nov3RrZSVxi!O3KIMl>U)mN*+q^4f*5SV0HvCBb)BtJ7!|K}aL`>B zK2Vo+@E?bY0(a9kbxVUqHu=`;{d02PU@0$H61Ur^fNR`_qDXLDH0}s&6@L0~ar9}@ zVRHO=q-pgAzD6f|8>C$Yoq-3?(SuX23rFkkS#SZEuzjWRyKgWZ>Bd(m`f&r1I;nHh)&8>zZIOnYB=~U& ztRx3BquDt^#y#PYr*N3u($jDY%v4(gHV>l#YSG22R5a;76&E`xBa8;bEw{Kv7yK++ zsnoUJT%OH*^NGX4mp&b9=0zcMfL(kAk(nR=$bq63!2mc-L1KI5#zLwWwp#R#c`fpE+Obd$rsU#EzMj?m3hn%n^CccoPI%01O=X4 z+MF&vm{`xmk|fzi%*!SfJ*_A-JbYHJb;L4y38A4G+`MI9GgEQ?VYAYq@crcp(MW!? z3~Ft+bxFN3eV{wO%Y6eYb$Or*ndM+NQ%-;LG3*3lL({9b$NGV-8>N1Sa{gy}!ex|Y z8$YGi>$V@XSxPX#4K^eDdE>IE?$l5h3O-7e{ZSE(SNY0KciL`3F2Dmn5GrP8rpn`; ze=CsH1hx2n<*0K~0Dt+K9uaiX$_L%p_N)WJ#)D4xgfGi{!$PmU)ZmX6A^7u3G2?Il6tnSvo<7x02kbMx`ZL0=&cgLPXj4bVu_kDx zr|IRkvVy)j&8Dt&z4Q4mjCI+psW1X7b6TiBjMxQD#4L=xT|PL_S#0~&-C}K$X*E`o zlh*o9Q57R@|IyjoTcC!|nnLTHh>E&*hjA?iAtH2&-)R}lN151rk{9@gW~sIn)iMgP z{B1jNP&)gsAniYR;vgSAw<8z;fODGvi#hi{gEXUQO__M?Kd3jD-!&_p)Yow@voD%I zQJ@kl5>y)CNyR|1MrAyD>vpXSRSU$8lwt=jR6PoR2=q)eF4IBC|hlg zhaw8v9QJ#C0odq~2YrN~w_d!wSl1JjqX|INRm*i{ou}X4x6hZhWj~ypKR5Wk&#p2e zev}`4a=UwB${bI#2r`7Ounh)Kz|G^%lGeqjIY`sdYh&T%xv1D0wPm8ruGmEDrOx|W~@EhCq%H znj^#>g0hgD!?>}en&pGRWYE&GmW^Xa$z)|A*(duUBG4`rmX4w07BVwToJHfDgs1Lr z)`cNlQx=UR#u41fNVH1^x=D{hGfrXuxmSJZC@>y+%YbrT2sf3reeB3Vh7PiQ=v$8D z-{2B5bkOZnxT^p9>?f1OL3?C#CnNv6ZYGms@DNKP58)aA{`W1bvr-;|ldu?4BK(~| z``**@Uvdd|kOxX7BdJA7+YKWbNU<#D$Na4^1QwA1&-;!>*8}2|J{;g4Sx5Yv80xS` zU5vznN1YFYZKnd$${mmh#7gq1MRhV(QZRKgmQy(O(NhxtSBd zt&VVx5Uh@XK)ifcg)w}am}J1T$W;>8E4>m2MB8v3tu2;I>~1|{twSx;J<4z$+1cv{ zD|<-?>+{PyF2M_IsW1NFNM#AG^G{kzsGW}>zbiKn&X06erZ-$VBYgoknTFt!;})b*(~z1Lo8UnE zfk=1p4w!(_jf&o}pl0FzJ=flH4j)}E-MqNs@A!Sn?0z{CZEtaD&~eEwwN}gOMAxve zO-870y``vDio)JtOGYJtMH0iqe@u3uQeiqXLQ0vyj*znq@l#`o!8<9;U~+t_0K7gw z3UrXMC$W;Zh$Myj2ww|5D-~D-syBlAjU-4wTu_2=z=ym~g*vxX84pLJXJx0Jo9qB> zQ~u|RGq@hF26<_A+h+UQRouh2(?@rP% z0<;dZX01dy`c5ujB56_D&}S^AExib5LbMYu^F<8Pq=_ufoGVZZ<69q)i$IDsbDn^SnGD4h;?cN)ZFq6n1bAR1VHH zPi$AWD5{+@CDdo#GYM2%;xkI*sYo+prPq%w>(SpE2Y?dLMw976_}->s;UR9m5RjHo zaNVe^>=|3`^5|jtL2tn_UAI@x9j@A0=(hfS5?oLtGB@QpxNly(A*atlD2#J)#MBpJ z18J<&X=c;;DJ=463nM5%V9mi!W~`E4lxR7oy_xhobmA!&SdrOptv-qnwG1>SU=$>f z-9wP2~LJ|=qOO_W~J;EVeWZ0d(3SL1YsK@RUAZ-<@ z+6Q}{OfWq3J;BvlWPLkFaoxYQL2k;ou6f(gs$hn-=ktx1$(KEUwQ8sObg3D0^GQ`P zxir}4mh5~+B88^8px}7Jb)q?;h*0JvNB7^EW$=|fIvW<7!95GAYG=fN9- znjoOewFVaw*{+$m_%M-}BnZcz!fICKy|VH&Li z1$1%jUQyY7^#(Oj&BX*puB&4WOKDTzE$R0XBBwfm4BY(s9=y6wz(A`$8|wV@v)Yib z(ex2Avk^k%2yM#dK;Ydmwk*+<8BzD_E`&KEd1cf*YG(ZR^})xXWAFx;RrlUTGt(`1&!LMLD+Smd!Z&{2MjiR zt&-JE>ZH!LUE0r&s=!4!r8rx8-&%}eB#oImbhDZG#1<{gZX%e~=!Wv#1a2P|_9wJy z=jG`%FNfX*&G~H1{>>H|;7152=CdJy-H<+9Rx)O6;m0Te9X!tAi@lIxS8<^@#82wA zYq(07Y@<%$uhps+Q3DK!oV>?!qizFTTDR(>tcHXZ*x%#KRST|T84bAd60BsmD%+^M zYDYT1zh3WS$UAlo_%@!pi$Yi?`{GI-D5srW`t~O)S~tuSaa@C3h=#|$(HbKf?V!YV zG+4|d^J{M+Fo0KV4+D9t;Se6UHpN6ac;>WC4{vW|6r!5pasm~qBMxMFp_Kh<&e`pt z??{-l=80-&zpLq_*pRu{{?&GHVyTp0X)N}+Tr^)Tmw)vP8J=67gDeX0Q6Xt?1*Kb> z>hLI3JQ{j#LJ+HXjcLD2@$o#2Z2J~CQQU`4m%r~gYW5AbZ_+%@2oWjWr-tlq17g)r zTD%GmBlr?f70o?v`US?jdBknkc%LX5l*xxLLRYIoC1Lf#mm2~;bQC|nzR45K#0)Vw zhVw6@cB6_eg{ zw>c{hUSOTc9W;^Cocb6Dw?~;H&Ak z`uVVseGK9&lPI3%PNu6V$W_HYl`5U?tu#wkED8SgHB9CM&C`-kuqO{XH7NbNYJOC& zdf+ZEQkEdLt2q8=9MH8aZIVV%Lx)WA)S0v{r8FBAO+|{>>mCVe>$8&h$S!VX=XP$d z7t%-UPc4rTiuvm|DO48n>+Q;^*a>r3t^MzPAD@%$zx?d7{$EdkymkhpntdQD50~#- zz-(CZXHG(8DEqg8UVh+#-}pBnCIV=_=Lig!1E&wA;+t8&>NKPG+jdGJ+lj$<5|{K$ zacm!nInNNA`U*YEL$>_I7kT~peet1M<88r&Ti_7{XCRQhc$z*F z*A<%TlcslNK30YFbedx5^Nj3|3$EW-R7Yly57Ua=eKc?KB|Ad5rNwvq@$mWqPNx2X zR$41=lT}DON^&{ul@#Oq-w=={;m+YxZ{XV)tsspebO@2B)me9XzGrG*ad0-*p?GAP zp=u009%?0C)nvlJ<6lzlCSF&hc@mEG!wMjy2ev}?*~(3#uqrhx#dfTlb(+|6Es8hE z`GuW|(27l~xgFCWN0S$bOD~F^I3V!574Shcc-h}*DArlR7f^^0(W0@{08yu(ByTL+ zH@*K;!v3dP1N^H8`@fI>QDD>1z{twb&er6=i}M2ikE*%|j&V)@Sl$3h{~E!+_Fu;1 z|G(}=Mn6-NDlWD<*8i3$2A~FD1oS3=2o$hTec1T$JTP&%7>ZUvB#Dv?PRb7n%8X%B zPtFXF64VYk{uC6(9Ha86Fv=_xGNlDWo_8 zz}AkBN|H{B*8(djLD8W9HRJzi_A6tfpB~GL&FE>B0h3RHCwkq>AXgWf!+qTnBe&PDYFo0uqwYfjkI!JXJZ! zmZyL=Lb+wshe@KMtT*V+)Xhdyr|UuCf$%}btDEobwAW)7NENzjzZ}pAc$U4@@5hCf zXITpffLf9>&6<9>{<>a()FrmvpbQcS;`Hx;YH4UhJe{>D;+3TsWT&#Cg9z~7mS8Sh zhCD|J;4yar&~ieklA%#Q&=O#L1wxI1{lreV$;^IA@tFzka)WGRge1m05kWG_N_`}u z(3SHBIVI5OqE`nj#sMEF;?mF&=>m{#f9{aT;yMUNP|1=kE32aU zG13tYcEq3-3bSM@c2BiBn_C?%(Xf~?T08|x!8tEXWcQ^X4DraH|3%q51&J0!TefA} zwolo%ZQHhO+qP}nwr!lkDI2fudl8L$`*n1G?ATwqBfoa8m1E60UyI{(MU6os02mQa zET|T~V?mT4+`x^IDiIM;T0o?!2HrC&{Z%}}%vv!KXa@&2HiLF^fz?`{J-u+39lF|n zzGrKE+Iw2Mx>|bnx>~`xy6j!dK6-Y2?aZ*%)(osZ;JoLzeCvQw3Iv%R*-D79)WT61hl}{x-$GxzqBFkNXc2a1;g}IS`Z(nJR)S6?y?$MyOAtD43QxP_&(} z&n|>^0oMFkbA2I{1;XlRU$YD$iWqnplr{yLj>ACx%99GaS+NEU9`BLCj;9jwTrM3Yd193&+|S7K<9 z=}{)(CAz_+!2pkKi;E0pbWV(ne?=~nysk-|T};JCgxLziYB@=4Zp>JE%<~U*cOT8T z?>3PrsyAx=gU@$Bh&@eEEJW1O3TXn_47FyoIAp|td}})kwnN)IV~V{YsV0l(_k2I4 z_xpNUYm1ZZd%b>qJEP0RQ@Z)zXx)$vG3(+UAbx5hQ{? zfyp2w6_5$05b2o36tafGCH38*+a~(jupK#GmqMP17L@qsfq0T?91=}l8bG@sQ-q{* zzsz2;h_q_;4`yrRAfhf4Q-NP@v$W+O0Pse4bJF{F-?uf#$@Lrr3`u_ysVwB}BNNtXf^R{vIoUcc_YTV3FO>GfVTTRKozGm`EEi zlH3y*8hnE2WOacca5&clI?lMV(CWJ5qr_=zwCzOFB7rx;ye+ze^wgjPM?bUn7@|AM zV9d5~G)Zu@y%vcxB)Gyvq+khOprPC8by#6SM_M2h-3b{^m5tMEy9v%3$2C8O^m%6L zddT$4i43StET=d~J?#C-oJl{wyt}KukzO*~^GNaIcF6+*J>=CoeCneye8;jTHwLPe z-MfW_g_a}*iEI{uMz(Tpbxs}Sn5TZ6)hEgUOC-vr)53u^!z|#Y{T*^}0OgsQ$PmVXETwlTk?N~mJ8#?^;~pu?jonSr zq*84W;|dK!d0~~p6~40OIvW7A^UgVkXe*nFfsTN)uhV0vy@IXPxtFD|8rs-R6!YKu zNG(?xeU0+r&9vAHpLYHautnD%4cO!4 zV+Y581DN0B%PBC24@?i1cO7t#WS1UFFO@S3^OcF!=nQS2jy2ioh*Jo5 zlK7K7C~+$sUNe^&FEAGelubZ388f9dEbwQ6jWA!33!6Wxgpx-1i8z;?A)tq;P_;| z(Dxl{DCV*XX1ldqwy}|yvo|WKD9n>J7%x>UO|Yur!)Jt0@ZKAETa@g0RxzPs{ckhA zDmW#lD-v2rGT0uC6=< zAkbohH?E+f_G4Cc3&$HpuQTwvQgUx3+~zjH2nh*E-~{XeC<+i=)wU@%mUDAQ zi!>XX4K3ROCi1UL+Mr?xGalp(#5kOqBg0>rUo^6CPJGI(FkPSncb<_ExsXB>GJ`b| zR;h_lbpp;x7d&5En7?Hu#I+6TUt8yOZ4kpvvlo7L?#sUH}bli$ZEYl8NeQjmvzg3$Ot3(_m*oV6+B+OQfRYXo~1# zP!*u5fZ)n>i!JGxN5SYpjcFK$43jE~HpPhK=z*7_<>JWqUQ&V-K*r@nqXOS=UtDTq z^l8(s**pp|7)=pXl7XHAlx7F@ zhM`2CVCKP3Y5JoN7a5P^{pg5rgQz5i1+=V#t0rKy{degP(F1PHlWrWaT=14@bs_Z( zbKkeO6}7%uy1ws^uCXXOKRxcB!@qBBYJJ?VX9sz7z8^<>X7Ffy-`suPL`q4;qI7sX zxqLpHKbIfdJ+BuNQDV=xpBwN|c6hbs$0X+5k1QV`Oz6QMD~rmw-N8Na15@hE38b7s zeQ6XCPWq-&I_dSz-iFbW#gxuXafR#jfO)Dh-C`kgf;v3G-?mGXjHAYX$rYKlc=eV( zc*_O@PWuC}T%T*$tb*ohr!>p&ek{lpLsSFTCORwD`ZYxVirkIGr8pA2=EBeZ^t@wQ)F1)XCQHm9(}O(dz_5l6L! zbhp8AIEJlN@c#iFk*7LBw`y+XhTIKKkv!r1pMM}LiZhk`zZJR)=6?Y;|E)qdakRDg zkEr(%bs4*DVT|5?br?2qSv1(>l&FCOQo+9A;s0NWp1@)i=O)%Lb~DYM1*#$_D2@}k z51H0Hu`Ma1X_`rs58g?Z>&4-;RZ>wqdxGHB;UF| z!1hK@%R{TCITb-LEy;QMLZ4!xaZdz&ZFNt8%^pFRGe%=(NKnE1XJPl$n zPGY|~)ql+8FEW|9KO-7K>@h;P;?crk*|51Or0~%%r2&tNS z&rrou?5e8YDt$mxgZv<9g#tWs<4qyn&gbW%YAp*G@Nz+tiBwX+|OM z*Gx}M58iEiOUuu|?2M1Im6MyBlb7G+U%br?ku&OW8)u9s1fvp@cIc zi~k|^6^uxp#EcYsoDO*_ApQDCF)ZbA81#>O29$d;d9!;IW>(rg{3}9{^hLGmL<}SQ zC&97bv15D3yhWJ^gd|u=NwnmQ1{x`Zr^lMdcTkYid|fiHp3wlPxnoUf*wi#5~@w@2krXB^F0g@$DCoFAx%Ra`tl= z3j~BxHLfsJ_9+?bd<`@&P?4J;1Jr~tbt8*vQBODP%E#jTgL*-+V}v+xWG__z>_TWt(Bva z^dogOoJiu^yh(N)I$WcfjWhXv2IfJV@^E&|)}zTWLQyjBxOklE4zbG!VYpp+*(MZe219+1Lgtu)TCC&l&L z9S5eMuJ|R_=o!&~oZy}^$XMyF|09JUtMZB>5l>hNV- z{-v2=n)T(D7py2wCl3-ej%LY2wKV9pia_*%?Q8p)lATF1On7A1eg@EJ1I$~2GB0s! zrZP=yIdkbdsX1^_8|Bma8>bn%wz^pg)w#X(Dy3kQeLV_MO8`V;bk{%OP_ z2&Kn#;VP3Cyd@uW3xk3^S}KPEWSqij)s+n350I(0pVGpCb|wD8vcpl3AVH{(GJMW{ zG737vt8~L|c@WZl+1gl9ZeqkV+SoIkbgr^%hfvVgx=3@~brUnXUT+ImRhi?BFvK!8 z62yG7R0t}K$dC44(08eL=<%mfwd6DKYt;p#I=w9b4!I2w=D>Ygdd5Y_8U5khj8ywwkZwvKZK%i3c!j=^yEx| zlrw;?TWfg2gxi0hwpQ8z){0q9pN-b0G-alhf{pQD8xd^&2L10R?|+O+AQPQgy8M>y zXDI*0M)1F!yhbhRQcgG$h_yGlFH()wVJ2QSbsIXNW0Iz8;W~l-JEZ=|P=zxg8u{ij zE76)a7nq$@J&U2IBygVq-yrC*+8p+0e)hzH_yBJvE;B-^2xNknv6L#NCMG6687V0z z`+Ga9X+N)1cA=r*3#=B21S-s zNvnx z!(;@GweIvuHG5=X$-!f3s*#PwLA6+*am&^qj0G>zB5hdIL@{wKo|37cI6RriVwrd@ zXy&m8?h}S;$uU(blyjCiCY{sH8PaNPF(JY1?ZGI+QuWEf+Hi}c>;p#U6B%(#3g1hF z0ilQ6Z{$B5#Kr#}TRrH6P&Ht3zAW-u7zY|EaPbB8L}n??Oru~{Xc7U2Y$L^dVCAeS=+d!- zRymdvE04W>0o5{*gO}~wN;!K2;S$zj9r5+&z`z=MzUO?eWCsu3#!}9x#F^^hRZ6E{ z$ATe&bk4J}UCWW!$T>1UzKm1RV}aVlrLA?YPNTlmb8b;c-lF$uXwIXm{jFl*E? zlX_-EdMZT%V)l|7B8$TBEz{o0!j_rYT1}$TnllRx%)wQWTezRj8uX7II)Az!!_4yH zAW&F1jNZXx_QWMMtMx>s)2AHq!vQ!>`y0I^b*zIVg)rns&IGmOSSACTj3s@Z+i(Eg zQi>5FO}lDztL=y@e~m%b<=#|qbrARo(8l&6^sS_4L`Fao1c*#zG*>byt0}aR)IJV& zS1KHBhA6U4XZ6US2}e%|?j<#Nnn}y{DQY#e!}IzeYBjsg7h};QGGk-GzB9{-M)yym6njg?21FFHO&-$hE^-d(7; z!>(oJMEwcX9&@(QC}85$x|k|QXMk2uf>zgP4O5##k+l*|&pGhd(JS90T_}f0N&=hM z9JFv&MsG1j#D%uol}3F4m}nLup4`6f{2zfiGe%i6=Vr3wzl8^}5ED=6Zt}0GPD3`o z8Re`v z5`ju%pb@e`l0g8d0_=M@FBOuOe+)$d`FASpM-}n23$@7~DX%%(3o( zi}C1R_!Qu!E-h=Fl+t+U@pOvv(ONtWyX=;>lR-p>9V)4+4Aj-T5NyKBaD0pK~uNyV}5Iy;`&%$1*;&*c#`%gc5- zi7FX3T3vVICH3kfUIw@FRmQIkyLBq)og&;8>IG64JA@GRiTjQ(jl0T=OR(vk6WRXB z=>8EKWL04^wNpvP)4#KsNo(r~N`O%ZO7nEzOJQ-KbwV0<3FwZe(#2Zw%Q%BkT;^zF zjliyqa%5DUibB{N8>cyp^ZN--$s1O%`CN*`iIr&?S_0lnYxHFOV9tfK3jY=9q`|%lX$c0)VZJ|(EWLT@=dOd1ORd^G= za=7GhkRnYUVs!g3UA@+Haw)|XGZaCkZavhwTn(51sGwkpufFzc4tf%xVklff(3+D) z!Nl?4Tp|icrHlR~$7Xa&m(a)w$CH zAa5weE7v_ddk8aOMQn0KLst5}y`DB8E_S*anN*qO1Eb@2SXq3MqBnnMxr?T@&!rv2 z^%qa&bc$D^c3#GA$nFjBj%ye03(p<5(lOI_W+?1V<~>t+uSo9*|G}cgt-DOO?3kq> z&Ux+P2nv#KPDLDo+&3V|$@<_HJvNM(>1cMUMs@lR=JDKoxJby_nR-JVwy_iCH8Jc) zop?A*9y?$rcagUvV&NuqUf~-{=3eAV<^$R^tV1k*L}O*$+sqhtOpNEl!x8V{8gEib zVOv9!i*hnj*v0yqO?6yW@)y9jdnE?K+pY}%g{Y@Zr7F@OrcyeXZr$1dKVvmT6 zX%dJv{nn+^Ib}(c0`oHA{GrS?74=mxzIxv$LI=^FsXJ@Wz0lm=6Kl{9(0{L2{)6TH z-Y)((c=;>A`CrmU7iSA=r~e(ooLdtn_WWI(@%xbdpMK>30A#uX{ojMk64DHgk^h7< zjs6?XOhpZX0sl`pGj}Nsh39v=6_Nn}fZ~6+VN;WF|1)^;&ve!y^U9uupdu_hg+B}n~(RH@?YI#cen5N>yeW! z4hU%}UXu#>NYq0sN@(`K_px`devL!sb)wlW4pj zTBzQCvOqz3&jLMEylg@^h+r}WJ@cw~LU~TW-1jkV*pWgZzV{_V$ONm_1QF8IwJ^sm zry?@~P2=$^r|ny~Mwif>On0ys`doBzO1S5;w<>1X6D?9FvDPWo4E-~36mLbtxb1b{ zf)ava$!h46r{%!r>@~IIXwP$%5+XGbW`2m^irFo<1*2F!;CTTcA}R(S8nN zZch_fGy-4}9Du2Pyb&+0&)*AYLbecnzgO69CosB3tu0FaW*5+c{v&@G493?>5ow^WXt?Zz1F z=z1DV`(^t`&#%!Wc2PuhWVs?^XPD+HIsG0KjnVo3;wis4>qV4Di* z6QfGRjG|T;SP$X1m)EUuO2{!P^BQ@%8?P0E!-y@448F2Q7z7N+-2ecSKct0|)8NzC z3g|iBy@B_zwbY5`M0d>tJoJVAAHYb7JXOIrgul9h^x|U%1L`#@_YEDo{{CI? zZU6xX{dnMS45QaD9y2+i6k-RZMhFsLmVu~EiPJvF^is?R-p=^ z@sA-(6q0wLCySSdmX7QBQ~SckcalPU=^~OP{K_RyF}VE?WDHae_WF!xvs!CY)@eL7CM$D_EwS{*`@(TeF>S0XZ+--||Yz=$($Mvx+3YvNU1B=NhcGdKR5( zYyn7-KML5uW5GWGUEay!9-6IEC>n}sZM2D8$u7iy?vL4scRX;1w83E`G_QNb>x1uC zIQP6gF&~Y^0__mDr zYx9Gso_P)v(2rC7*fhph2G2|2cdkrha^*IvtA&4fM_RN;rR6bjRB^Dy>0*z@2?u%U zzP_#Ka0y04lX^PD+^R8d9;B*7m{%~U%K8Y{xlWVUG3aS_lPXD)l=#qX|$xm4vrG?Kd@?m!nx9UL5* zvI3y~S**IW^79Z=F36(lmqrY8s1>3>J7Qe4T{36vpT4;Mi?nP};K}`wsNTfZk?Tia zoh4i96p%7f!F2%@i~}L4Cqw#pwee|wcQ)FpdTY9oX=PrkuQTE>dK0*%k;E{_RRum~ zcgghrt;yhr$BY8|oW(kZxX!GO+%>t?uaeOx78OHkvj&&im~-^!wEm9K-t8~>2uCyk z9ZOM?zP7Enj=XemK1D{%mQ)E;i##X$(Yy~!7_cqPiCSW#O;tUBRnn?aq5kt(I6yu6LkTp8gimIxC22Z{Z6etW&fdNF3-&Kw{s&qk;N_QII2pw}UBj ziM)DvQ~v%`?#`N1v0`a*n*#*GN5Sr!mi@o}%dKmjZu&3#v{5{*3x;7msS}Iv7Om?> zA2h2Qjx5-0JZT{#l5@mNqcx*;T;*wl!v1`KrdRHWa?dXwk}yD~>I;**tF44S5DWUe zcYLjtrwj3B^JiIOrH>3vOC%F;;HCK0y{F~$u3%7Ed6*#$(157U4pvfIf4p0PWi?6% zr|D8kGqI|yx!9Ig9)LcS13bCmXs)~DOkF20iNF+xkk+9OR#zX5N2QHH^Hpu6NSeb;spW&$|@HApwk_~q-c1Sg!8hDp3Y2aN_#@P9%-p5AY;}{*ty?|f4>!Vz%Hc4wmV*8_MBPfnUJ7II@=4?-Ej3F=E#W?YeVj~I#w&E4L9qnBmYHM_ddRKdt zl8;4MfMYQ8-}(R%<^4%jflCR)PHGkrnZfYWkspd?4iy0f7!XPp|jPf&7BClJ0@ zV&P}FvMmC~y~;43tE((Ic<4~0sz_7n31_T0p0H4`_+7~M zByO1p+#WR6MlqMnf>{z7NS}-?^o{zr0Yn@e=B3Eqdl*fOL)E^U)q`jIL{P3N7MvkGq;0>3=3UdB$v5UQKd14=LKqmQWZdgQnN}qbT4n7D?HR3iS zX9lnEAZlANT534Gy901~GJor3#V)pub6W3nMq-0~`De9{;4%;UVJIYt%@T)m(|MGV z!#owsj#J|M>gX1fhgf{Uf!eqWu{pYK@40XSNghH~fKUG|s~e^K#D2vD>(*XRHS)Z% zG*N!<8*=Tql5EbRY-MPBE&UbJKFw9$%oF}tJoAZo_)dO{H{RHR?&?s?OinUllbncU zB;~oN6*X4jWq~u_^&g$UeEjVx|I}~dmJfe7^_Q_CTbZ9Vka9}GV)~Xm$5$_2%KwP< zGM%&iw(0Lmv>F zsFavJJitE*8P6W7JVLuBPNE_?O`m!3Cx+SW)UvdWw4@eIxHg`zKNfW)Q;#1Nd%ZsY z6>_7#7%Gw!ntkuWLq`KnM8#U-`XcvcX{W63A;-zn@+ULn{gJNE>pbF0FX`L&)k&yW zZ!w5&m^CHzRt6EvP(RW}38Q?iT^D(aMW@cGhqm=MEd#+o2qx38tKw1yv31y2%eqZO z^*XQccn`@JJLdHo!cUUhHvF1att?~@HO}G?^G4`|TForP8j2%|*GxHQ=;68h z_AHoh4!FWhj=L1~i3}UP-2rfAW<_MvxKnsU^-$D;PZo@^!T5NjSDF>*+-N0Va7N+a zrr3d9aS;-1h+W}b1X~jd57YudtkWbl(f~<#@*<8%4)vO*xhn!{`_|V#bw~Sz+n63o zW&cWI%}u+co0dhVun+US>VW-QMrgFS1Jz+3Z6Xi@8gXUb(K5UtqpdFxtvTy(8D)~c z%||I4^V8aR?rAfC@({2s<81TX`J9k3#G<$lp)$0nY%MQAozPdJFZ4W%{bw$O{rYep ze`yFt76;y&z&#!Lh}IPU0p}R&t?xN9W4az_$ zH3%!Zy4MaagmsHUIBqu%Mlo({qc^hXBkrov=mpJ7q`Gp@?bvjH9nJAD(LDNQeZbtl z0tlKYf4&o62NBE$AJaN664g17W0p)9bjgZ!!4!WaN0LQ;ZUAX6`3RfU`4lBZ{Wn5g z_*54ay=ogF1QVK&+bd9uQ!hUj+mxm2icv?q0vcUO40IYP=LpB~2)JL=Hn8u=2$uhume+M47VH#RM=k6{^LZlfjH7mUBWo z-vn!jF|TeaoP)wZSgx*QsCMJb2`{o%4=5mwJ56!Nku|-Kwc@tzNh}I`+)>_Br54?a zU7WsRMSzyrCD^#@RmqvsMAE1B^wolWfc?oZkRly~SsH}rN#~KYQ3d+I+66ag7X7%^ z{KB$Io4>PAIYxhynZAm%Yg3mh#~CzakNzA_b5okBPFVm8*O#@PuR&nsmp=C3V7$H% z@3jpQB-$FnCb{c*zDeN48dq$zL|U>%HdG2E0^xuVrh$OxNwUbnFPQH?3h@e#!Gl@o z3l1n{E^Py{|2aTR;EJb=9T>1Cm)+Z9chpr`qGkp7NEjDBsP~t{bFcl0eXxxDhNEG^ z=2IB+!W)R1^OA0r&r_im1H>%wB1%#=nDB{nTZb15n2U1vKS_cmt;)$RVPd&+&8JK) zKplxgTw82 z``)EFTg7HOzPX7Z)Nr5(vt@5@IPwJNav-sb-w3^cRq3paIfyXiVc)DQadN*scNy0k zqJH8qnl#74rQ#ZmzEfd=uc|duZGnvP6o*!9p z0F20ME-efv(XQlurolYe=_LaU3RP84_;wYEI zKjs_R?go)Tnc}-ZD6i7m0VEV9X3kSTgv0dz^J&8OC zU(^cRf_R~?!P7=gsDGvWv`Fu2w3kTLlO9(2&ua_cC6XwLb@Z`{Hym#>U7FF|y~5Fu z-onQf2lb@S+X7km$@E|JPi(!OUOjK}CfK@Gv5#}w+)=Tgq0eK3C>6>QhWx$o_5t=$ zhLOfQGIXZG^p{@}Nn|lnpQ9*x1ArbZ4SO<=Qtl3L_>k*oPO>|NY*ng`%+d`2ADB}` zcGOup^Fj+SBnt!QV-I<=`ay5Bywu$PaorTeP+JWt`O5y z&kj)hT15pTO`L77^K0|V7o`U30Wube?lcCFy(35w$6phC&y>3rMfS-M2}W++3|oE) zAbk+ar6ib6%vJlGVmr?I~jwq*cM&d9Ci4Gll-kgEg7BWW;4CplZtA3dAz|uX}Vj~3^%(1HK z*X8s3&)}#H&$1#7MTn5sE-sH=-scGAtXO>AI-zYcrdkX$2^_UJ0{Tv84nN8w>W#Vo z;Bb9fjY5M+u)mnzEwP}%^qg3A7%)zBjU|s@bx>KmLe5w2Ova%HQKPsx|f{4J|qK@L1?gq}d5^Jxfv8vp;8{^8+$@{ars8&X(Ez?)@E^ z2wstO=oaxGL-vjVkI9kQg+2!!{kzo0;P$W4VZ(c@98<^Xz_Lt?51KAxhcot#4F8$N zDHhIeR-2yFw5`-2e#{+a(Uz(85<#RC=*IH&O)^TNO6E;64B2~9 zt+d?&>rImBh)zWjjZ`=s@Qo}i(SQ@bUP-{pa~5sR;?yZO%jSDx!&Ag{nT!{hbCK`N z`ji z2-pOa7VJn_}Q+PN?`^pQ6np7TK~%EX4{_fz~8d3*wG?yFsXR zfOUN7CQL8|667$$L289Ua+^4HB{V9-ix34yz;kqKi@9Jf2CZ7h{DBK(tn6OVAv^lD zKj{rw$fmY`RNkj+%n@0Gckx`qg)(MfAsWu^$iV<#XNPExng_ns#j@&#@)Iw!DIpsa zwGVMy+mj}8dMyZ)Q34Htf!b!9pE+Jh;N#%(1a;aq4i7e|5v9)QTOztv7{;=X7YQsQ zhc;8&rBk6Cmb__H&$Qqgm6=^ezG0#tr1dzbrJu8S*6e66!M9j)8^_~f9-^}6&^%rn z60Y0EcvvLPzniu*egn}&4(K5LFwyHhiFo3D`3hyTu22^JPF-W8;Gq(M{f}${UTZkYlj}WF#nJ9-E z^l}y?!^&TUHB{O5&K6w8ZkHfHZMjL3>&b)ZNwHTG?SvUYm&Q`(e~iJW3$c?`xeCWC z?;4^v^}0r-o#+iSR$UGJJ0TfF1iVnKdlO>oAXcLD+vS{Coa=(u?J6+>xsk1aPZIA) zl;j?$W=`Sy>lRU%f?WY6*CI(yi$4%y)y7-1gZ9=ALoT_=6cnApf)`{Ay$$SN05Bmze_mrQf zyR!D9@ZtGYqt884yVj4(s<1GrF*9yEAM=^oHGs}-!}>R)+bXBib$W@z9NO^l+K;e8 zpzab(cg>@)VX zwbR}NMY4=Q5lV|8%t)ej!KeIl!KRPT(4(v@TlD>3w5u)mmi(TFj_ZwoL#y=BA&UYW z-aKVPesGUbIQ^o(e3Hb;DzIByI~U|RovX>|kLCb<1dNzR(3g>d_goUHt$vE~NFd!L zy{V=#NAawGdgZ$hUlniDR6R_eriowJCwTfV!Z^<(YY?(IESsn>(5T{-<0**){~l$a zIs0@Fg&*VHtGY`Y;913Oo(-VU}&+NncON{C1*PHDnK#^3ck zR1P1)yhFU|8!Os8m#Y?PF7UEu>|e(=0n>Km&zGSLWill;1>3)DSh*%G% z8Qwj<*VqnrWb&ev>sEOe0YemG$1dy>)OT<;?poxX){(44!k#+O9udWwbvHFo0Q)R6 z*xD6y@Fx-m%-;k?Nf=+0?_umcFaa5~%q0ZUA_ZED=8n|W_VW;>IR(^=D)3P9#-P)Y$p2@pIC}E!Q2=blwnA}F^>}c)GTiAPee4&}- zHEn8bd;IzN9|oHKGaw}RpIcKqduIz93r`cr{~a3y0Vqjy`CXU%`}}sd|07`hU#|Z@ zz=Exy|HoMH4U{yaBSWP^InI zifF;9@{lav-Mn`FoVgu;US$_^B3pXM;NkP;{zND5atV&d(6WhAtHlmq_-n_|`Vsmy zP$zYNOOd5$Sr{QRCM0eX7m>bfJ3b6H6~Dkad!tdRrJ5BhVzLd_@Ru7?ep<f;AxBm)HQ3muw~5L}+EknavjMwj#!9_Y$w3-O0p+(_}QrL5!LN zo!u-1&f0?6r!5s{&Pcl8Ci+MIaN!NX*(;0x-R|X0phy1llC&}O=qh`2GcmJsHS}oU z$re->XUzB^`kg(?3j+zRHo)@P!W@pz59M40kDK9y2QcmeJD$po+95tb+fu!rm{j6> zAR53?d|Zlv7Gwda3#M625o|Q86dNt+serPR0_Xny#7!YgV%EYsgrg0Y66N606IMe7^ccEHI7LhG z0-9jLgp`r{j;=>{)Tv%001{F zRIoyy+q$R`acCGe=eP#blndNJMfFc+rxRb-0yr}G;(%s8F2syDZ! zC?SUv_WsS8BGb!~3LiIhgNS(W7k=F z30yupK_wl^F`4bf^?RElij_%B8cK{t*;D)6+CxCo@TY%kfR1#!_#~7~+_uD@G~!7D z6c;6zM)-LEd>q69rBU)r35B)$jO9lqk#24Xy*Bpo#S8^tw>bo2*XUOMmk}qQ!(1otUrtAD9hLJo=@4 z$%}lv+irx%sM!#<-WQE0bhhnSP?qo$eZ}Oi{_z05;8L|GeD78M+7d)m|w_CQ{ktv(@O60=u45e1oWVp{mvY z98S{>|Ecz`Z!Yo8sJLv?C08Q5Y=M>JK-T7{OLBMlL(?RRXsX`2iQLcZ7CX7J3cFyKBe$R=@-deV9XY1Y89sgFVy=irrn!DC-bVU=_ zC_7)9WI9V%EZa9kHC9}ln>18vxb@&(_c_r=2JwYSTZx)??Rd@XWSB$imH}9}i$^CM z76X%OczRP+>VLdFQ<^G($B6ZKYWuh2$^`*SorMzUeqi7MgyNZkRg2YsCc26k<<4+` z>9YpgC3xajB%ph;q&nin-s-h=x1Q^9{TY(Mh86tah(1(6Cp)}*zmE92RMG%19tPU( z3$-%I=>1u$8nx1DlN4rKl+0rFlh9j+c6>HaAE6Qp*}8h#6$WS{oW>oyNzv7zCY?RO8iQS#r5i8w7SU8su8$<27N~`bVdu(eY?hu;7-sA7bJi|9`#+#(1Sp zMf?@+G{OFt@!|h+@)*^qUfPn3pz^LT;dx!%z23OHqAwyq&WT3!5dHx`FyHirBq;lrl zs}%D?OUL_h&_xcY+(+i?_nGETG$sLIA6?`>AE3^E*xvvgxC*Pww%=~BCZo%C&~CBj zgfKVY0A2aU9gV*44mgPB9l^aN+^luL?HysAYVNzP$6qi~=kQ9xrhvrrAd_5Dn-^+Z z?VTffF}~8PV2hYtULWnygPySJ6A+{^Fo2Jup^ZgaL^9XhfIZ9_ohu6J*py5Q*Ac5o z9A<(QpB$7Qz|7eZwC8mk$mPBa5Dlt*cs~D{LJX3CMQ=OkUVA?i&J!q05Tt3GG=Ml` zbClDe0bKd#3hbECYBiZP8a?A*8+}&L1x|FD{$_B6zmmTcpxzERLrA2vo6~Bb)dg;P ztNy0HRP=;GJA~i&pB+6@SNwqX)Qu2?jzZ|$P~ZbbtH@zg2(<~T;-gQS(T6rnwJU6U zb2E0F=QRrTfBXBrS!WK#Q0Tyc299$+ImsPauzqMQ1+z#mfMHC^0e%OW(?0cPr{QxE zc~#*;LQ`eF+La19Nzza=BDK@pv!M>BS$ln!sT84-=n4j(0@W}jwtvYdKLar~VH66g zf3g2ifX3*I7Rl+*`}kmaH@$J1q@Qbo%XpJh$)6VAPvE4$C<4{570oFFZP9`a<5;1- z=oslwijc}TVYb|+_MwMy5b|mETj&wyK8iwx$(vdj zhN>(7SE89j2p5AJ>j!XOZ3jBh91Gep+%|%%rTNYTLv4o%1j)a1Rp zSiC=RZH5yC!r5&Czu8o7F(7CKX=$|nsYjNf*!FY?g=AtNL~C{*x*x$gq27O48ExmbUBvp4Y**u|Hw3<8-QW4R>MMC z`w4IQCjuqC5#Rtn>SoTX6Ou4g|Al!-ci#(rISUZ`~sy=VY)aiXD!Rr=@=ra>^(w^ z!z(A(^swCH@x*59keK3d4wu~J<>R$B)BNlABx&-31&=9kNO87U5+Ti$147cK;1i~d zk#`6}%yn8eLh2^(=oP1J@4g}~mOcSQMyJ5Y>$w<|F_eZg>li)2i_fBJ)%3h}`jy zT>H27TI+epm=hY#&d5I*#l{7XwCCaj10J`q$H09v>Q)9RV7sf|%l-G{FPLI*93`XfqbVPm~W{E&uB%Sf7~J!=Z)!|QAQroAF}`L7|#4&e{vf%f01 zjSB=5wg5r*vfI^XXj}v%LgRa4=S<}^-1aj#rH>_#6$;q~5$Z|lJ_L`Wf zJNIE~8%w%U(CP3+lt07QVSv8vv1k!{ALXfsn!!S@g}PJey0_-ZG(ytUvaq1QDD~lT zV0&-4@2gPC$0c=-9YK@FyHYbHyhPNr?_t;8<3OB=htvtr+01Q(+Z(gC=cwtD71Mb5 zI8?A`6$WcJidzqbT|jsf#>W|TNNJh|s?;D&!uKLu zo}7d?evg3!iMl5IDkj)$k$!UJQ27^+|H3A|OG*5NKsx!Qbpv^JncSWfQ}iu6xnoOv zhH9hz!euC8vFGJqhG2h^&HvHbiu=E2o=&EQCjaNORp`9!%;{%Q{LdTj|F-!5C1?GA zZ)FuN52ySu3F|dm`x|93XSo}JbcA3C(7&{>{`|oI%T|qToLT-|ljmOrh@R6dj{eVE z{ht>9|FYF_&S5gia#7NM6Ce8Tw>kx?F3%pIumu1L0N?}vU*wYh{i)g-x)@uq{Kt6n zw)(UkHakMkB_-IsVX;ukH4Cj4Do_O%Bo#{>^rR&M2AJUWjFgDlQOJ>&Qoz;+1n^}ga_E(H!}{M?7nmJQiKYLu{Qv{EgtT#iRmUefIX#R zY}ylO=&U+?;0GLbQ^CJFLS-(B2mGYyov>iQR)OJGzMX=E2$X_V;2KwhKpBKbDOB zuvEFNBV@92Jn?S&)k>()oFStt0DmNroL3+rqNU%p0{0gYWmXiQD}!hiLYS)DSP@2q zMi5GBmbp(1gZfJuX)rM5MKJ?iddQjR5hhgsxP=)#D2=lV>CtallV6Gpi!!yC{LVT!Tq>IZ=FA}V3>*mJfvb(Kt- zV>iUDC#p=kw+u|5L8PR?YFEn|J3PviGE^%xu6QSbUokDnDryp~<~)pR)dl4x8WhxZ zJ{Mu#B?U^-2LXzpM`RF8ib$Q?87AD65mFU3Hw}_toqt@H&9_3J&4#p*uv-eQ$OAhw zd?jm<@(K>%2oJ=tDvOm!^h(lAdoHhey`1?qo!UmnY{nT>_XiM0EP=^yU+8a+7uMUv@ zF_hBrVbH^02aE(}jRQZ9v9(Mtr~r-;JIjNQowI+;g{^l3i6D3|aJME8b*muum5vT( zOBn}?U%;Ag%k$;^+*G0N@VPQuol-TE6dr7A-e7-JvM(WF*|21V#j-6S?XuQ>!XibE zw}jt&eooTr`RHuB6ckYD@zMDcx>=e{MZDGgLa7n>K&q6N3ouP>K0bX(#w3T*n#<1> zA^QrGcwYj)HoG8)f8>eq+~Y!YhWGhP;#-mOuk1w3*xJT2H>McPLCR`mciX8&@U&vV z$G73pEu3k-F(#fO=y6~?wNkBjK_xC?llJp-SX z7pj?By+lw;t6L6b_jv!RhsbfU-0zo@_tS^IvXb z|F_Z8xLJMEZc_|FH=!e~!IaZ6VMKPb?M&7}nOjz;5aQqw?MM;ytUxdt3|mWQ{8FJv zrbda}AqS_)kK%m+{`~uq{T=W+OEVz_m>)F4CfYn?GQE+xvC;U?uG{5Zbua#t=-u7f z2dwV{w+PX@s|d}|7$SF!13@Q|+wd4NH{N)TLJS56QAK#F5DxU+gPg30YyGzE+~qs} zy9@3C%#ciQ{6tlk#L)-GlrygfT3t@6MhlQ~;U9N^Mp+R^gSt2f&^JFx3Z49}1>WB# ze}?`ni7pwG5EhUz%j*5uH)J9AyRCw!a7mzDh%mEo+z5jUK=(m{kUw6H1^*$h;Xo2v zkP^pXOhO$1njx+Mv)ctFISe;O_kfb^USg4wRTfFj<8-s4x6sUnMSe>AxySba6>5A0 zFuXyLdWPq}y#~ey>u-`muO6UGdMI|%6M-&v5+wOm*Upxy?P_3G&$LQ8h54r{oHXoX zdO=E=1b%G-w1vrl8eM-7N*vRtQ>QU%H4-#(ZP)FEi79}}wi+$hzR}RHG~pKWPjGuj z<3xCOeML^lUG%bJNda>tP*SHQqG!hQpe`rebWS?CBMRPx4(^x+X-RZq+yuv$s%e9~ zTDuB3B_t&M%m9g$c$1ze80jQT)J;f9FS+xVPW_n1k-+E#OTCRf69z)B$Hb)Wdm)o= zgG{9%RNFKTKw0T$N%cG&qK%lGXBjHjVJiH(awcgu22$hJSkdb6CRj49E1m5z`Jg{6 zF`lsLs|*js;a0|Vad z#iWNzNe~eMj^jAxvBNks_ko$R2p`_Q=%^&H=j}R@@ty8Rq{8h{$prs*y%7hxa6H& zA@(+6Cvq&WGeluK{>)R0%Tp`tT8qr8>_)9r(9Svjm#37qt(7Ouz%rX$c-u_6fNQ#o zveDHjUi2|`Q&;D}2ipa5s(7vm+nz-o>};)J9`s2Ek>+ux2ycYa{C3~S$jy}timBYX zVva3UMtTuF*mA;KRWyRmln;XyQINIsA%rBIkP#Di!)L0f&Ys2to&8+XQ`*|2N+rwW zVjv@F(_O<5>~p@C{Vyz{pStL4fF;4LsQfdJ_o_otJ1hvkapoCCzxW!N5OP)0iziBk z0-S0=!(5T(Egha7uqXVHZQLcnpHy6#9N$A&(D;Yc5Jc|}KsH^Cmiqe$`mPgwq&MAL zp2M)+Admg9!OMf7Lh;7wFLCrCX!O+u#2TBsMCaQWkFojR!+NF&!%Z-O7Szj4NOv|! zcRWls18uls7JD2AG;_G_xl;q6vp+KISxsG|PQo_plk;-4-FwO+mtbS&1nhz<0xOf; zwh3o};X=h~b8%uY%A8F3?JRh@XuWOz#<<8Sy*iF_1pKA4@wJjRPS+*B%Y({uOmmzF zda%!X&PxmLxD53dWt0NOZ3jcXSxO=s z`B##fqb7XY6JV#gp)D5&r)IU9?@iz}H7~!Ulc@8(lh0opCSNxhK<}|;nK_KPS2pAf zrEFXOQWW_ya}?77^*ol_cCiG_;{#nw!|)qC4Af23RS7jB!z#4KDYdZ;F>%gR)G#`_ zeBO)!kBt!W!gG%eV(i)xb$Ys*+P&qodncB5EdI=aVbUypnVgu~5p-lN-q86Y)=!2U zo@+oP)g!Rga<@C>xPkdIX>jEsDS-M3U?$2CA~eEd(o@V_;;Cg^E6H<#2rtSg>XQ6h zqK1U9;<~N(Y%!%THR=^B$a4FVB=X6gI5gL(@)(pKvX=NE>$Bs{f0A|NC|wxt4J5lq zy0Nz5K=h?_>&4N;w%{b{#&;wG*hmmcN_|W}$&KQ#F%1RJ1<-U%Fg4{(5Z;9@C@Kerh5ci1pgyJo_fK9#O^Mu;;{#F`6&Mm9d=ZlMY zqj0T!vwUJo@W)dNZb4C@F5c7demhjzuOT7L-^bcN7MfNR%|#jv75-abf@D%gt=fe) z;-d4Ss^>KsMRgx~=Ds7gZ3bdp#0`eh6MoEX)#s_NRDe`8HwZ*_N%b34qWh*9nkVEt zI|b@5nWTZoc-pZbhnXKny7$l zV;2VZ^$6LS|HU@Efk=@smAE|~1Ls*m{3Y>ExE3QI#SC^7^W**e_L>VxDNj( zT*E&u*pv{lj950q{1dM2wV%NL4X#g4(!_c`xI3=}1@e8q+W6pG<+)YG+Nl3H*MXf% z7m4r#XSmNKWU|YeWKr1l`nb@u?K8_v$l>qJCX~jG{J?eozrnS_wA8^h654fiM2V4hRsDT`}I9w`~1M3&rVp=Tcaj&VZmUEGw$WPjd zj#!IQfgudv>A?xSx|^>f-%NjJOK|gz;Z=3wQN+ zuWjPqX0#uSp?Z0j=#KTA(47hUE|oL(BBqHfA9(nb*RYH}#25y5diC1P{_?k$>lK*; z3AAeTf)1`yw#zwrgEBfYxvq<@t};3t^nD5c^VqOzHUKqjcl>u3oC27&f50`(Zo&Q}gF4{kYcnY|0qYb%oGez7zb|wsLfkfc zP#@|9wJfqgi_D{%t_Um{7LgUx{82jd@f0SajhoM2nemuVwnYf&AnzD}MNE#kfP=)J zXWccwn+RV*l4+MAK4DEVnmnEYt2{Sd0j3-dhu@`fOVW~sNi3o+fC<2*&lx4lnvBtS z$UM;i^nsQsXnj*Sg5!S=(0hFMg$cvdEDKaIf*o2=W+E(VP-eo9)YaH$FVt|#w9A&7 zp>8IAP{xs0oMM_EH7DP`^Mbo!yn3=4gN}!TK(Fi@oe>cQLt4}#X3^IP zdbJ&2queN(Z)zM2>}$%<4Sk$VlgiK5iu$OL??gACTaPJNxXn`53_0T4Hxlcmrg$m{ zk{#a@Cro0SoGPA`jgWG(f+dsSEPEnglu{rmw&7&$#ZTUojZ*5zdSa3-jW%Z5cFDc= z!%{XMw6Yan&VVSfq zbT+2s!C(#*aZcKAdTFGPb}%UhQSV8joUMy8$=rY!)R_Os0vXWbl85Lr!A8-_awn0W znXs^-5j~81P%KkiUmtbpYkHOP4x`){eoMnK6sC+mkJ^@o32&XYce9*d0@@GO+(XA| z^*JX;j@kP1Le2qkEyw)|QHXKHskRm8Vy~>18%fkooMk^6dC$OtSrIct8MZD1&abv2CMCZHLt)#v=~{G}S$rut8TM+ z@thBLgA5lu6Zc3y6&_0)H;h@-**N>qX5sXoXr1*>wC<-NA2*QT9Op^sz%R{zdxRrz zo}M>M<2RReY^yfP32Vbb3fv#05pbb({i;d;cUY_^glLC}9={RXHl=X&HD7NZ+9f^3 ztu3lVu}r26=r0+o83bva*S#6&Wc5H1!Sw^3_aBUmx$=B2GK6_O}u6-4~DF%{s_gX>YDVC-f9&$ia+ijygK&2sANK|jwqFCLr^EH%yZ>CAU5co~6GMcWEt>^e|BPsUqMFt+h$-fxf}$Zv4f zQJD#E@|tvtvJ7V_2iddFxBH!FHDjqP$X2f0iev52Jv34h7;370U_8%N1N5A78p$R) zE~px~DZHuAWd)|U4znkCopS{ap7nwqyp);Hv83+_uiH{cw(o#Kz4`f0#d0WfyoG6l z?EmYmfnd_JWkK4qdCz9@Q^yj89b9whQ~a#T(IiFwZjh958KrOnet?o{!;QUmW6j7g z(4Jy3^8^3=t0}tdLaXL0-^d88Lgs}C(?>-P!}L6?MuI+&8vvXI&`$_d z-|+PvW2F@Y2LRihhrqt=)V?XzXIB5#dK>P$qyvCs`9PH`L3IV*JK~S}$@^Wa+SA3_ z83%ouYci=8=#}XUO?+eFjpi<|22G>syr#>)_SW4amLxy@Aa;-Kf2sEW$5G*A{2$l$ z*Syxw*rV~gZ`3DwPJjfzQ3<)rF&KTsCzV9QiPsBo)Zq*Xr40nCn%9D#1>+pg{XSe@ zc--_N!9_c7Dws?o*Rpf|@w&;$!P|qx&VgI!(I;7-PWyVjIlhH|Jvo9jzVhs8r%%XK z9rk+Q^LFEVPw>U3N25glZiwvAIu^yOk7O~mN31ik_Z*MZ?g^920f_mNz&R_`W8m7d z2U@8%XMsGsA6tc4*LznWb05VUIim>=$kkA>_)TokHf!Ay^G7p9y^4-APeQ6iCePiE z{WLRz6jPyhr${nunsbwEIie$8!XM_@J)Eew#>hVg&<|K%8aZYd6*0M|Q0hk+)EsVd2;t0|+jBDgUdpueu;2p4+l)v{@*ue0OZ zg%bdWpV!lkyuF{#s9%T2$MeBc=MR#vXSQCwWv=e_c)eUqTYGqo)9HO@e;qGh+Ji>{ zSLR{+wNl3U5ikj|EiBbb&(H#;v=fwV-Fi02oG{c}mBiR{bdKA5#`im{g`i&+4UNL(R^jiS(H^Yc$ne2F;z0^Y1Ft@XRtgV7ipm58fLrwKrsD_HWy8$4uH-G4wsTha2fHGKz zIq^*2lI_q9N0csIoX@3?$ZRO@7vdO=atP?#8lgNmfQE)pJVTdIW&s{i!cK|^?;9XP zCJT4Y$99vUu@j-cWbrMf@>VNt<)o>_y!ah%(APw*XM13^+#PUt+Q_ndZ^S0v0+DtW zqTXGj3Y_JISxWln1-D@3ZoBFT-;8qBPM}yO?P8?tOph}f?ae2o(xpEdY+4!~&(XKc z(-HqhOkGfTn@Q6PrkPw;POzn%+9>XKQr~&8pwuE+m-kj z815ccd-VLyER!EF)Zq&0aK(eDVnYndYgHJvd`Kpig%4gf|7B*B@5{+1yYW5yeTR{0 zM5JG)NuEH10L!4!FQF%WtA{R(o7A|KHqBs+fYZ&AP0u zB_Ckq1&biSOO$NB4v>Kx@JDchOhTr-{Zk|ulZ;X5V^SGz4P%Ct66u6yfQ9cuYSN3| zK3bsAl8)6L&@iSr<(Y#DDhV|hbwHLZ^KDx!Xt=-fBYx*`G6Oy-*^HTTp@qLQOl`Rg zDFfqgfNXoefu@>+#gb>p=#gDO_;8iS?N>M^bE70=&?7GSc%;*^E*{LUNmLoz0<6Mv zCD3Nk#0#N}-YwEt9KQbYcEe}ya7p+1Hd1i;zj+Z3)PTUq*PszvFHI|5F)6^yVzadI zOFP(~d%--T+~^N#1f3w5^($KU`j`lb4}|LTxe=ea;|AVgxyg(7;@5$QN#K99Op+&; zD%_o_CF8`FVP#hu4fdTakQ{pXM-yKv$6~n`L}@ABW42uF^M??+LI~TR$qQ5~T%aV6HV>zDG?*pC!bL%s~wTmgoe6-nB#0aqhuC ztv*_}NegRa;7kUlN<~(A>c_2Jr{dGyKkeIZnCvJ?-9gaJFbG$SJ`8o@fP^-pjUt@q z9RT&j5H!%3jiGnr4~g$)^x8rz3ygmq zW|~lF;BG70^laGwU=*J>7$Z829`(aL9O|n%1hCPb$1}Vv4UMw3^!2tV1l&-%{$deUjP^LQl`|0^dqyP@~sMDSa z&oDDNUfg=mN%JRNt@^3^3mlyJ!mREOO*=%bs8QM z^YVS7#J@G$C#*H_B>-@f)$1PT4qOvOQ(-jKqwBEip*_Slh$wb`7ZAQLWaUpmn(lUG z)*b+@_e?`K6WdT3lJ3q_F>93xpD5D8xzovggSJkIS~Xuxf8?}p|4u7=!QtoGy~}ue zFUBll(sU+ALUN(ZG*e!vMrPZUulEH(H7F(mTsHqv8b`U1*Z?s(2wjWwuP5YZn))Wp zaZ3nB)KDX05N?qMc$$H^(#~mv%CzRg&}-99MIWDrQG>!;nBc4OG7xDz;Smcvw|5gm zBR*^ZF_i}c9Y@Pf%C6+g+5ByJ(N3Nf3H} zQRu>g1FCl7`PS!@lagX8S$N9HZKUc$bKSJnR0N-^lZ^+=F%ThfAfK{Y3+st)x|c~% ztRZPy$Iy)IAXYZr)SPT35q1q?tmw@muuYlSQGac(B}?{79D6CLCH&1GP?O5mh_@Uz z!^OE7xXO1i(D-`z))bo>kIGgNe+Zs2?Fr zI%wZ>Z96@7@u7ZGjtVC#67ZLWmf+)??xFYscUj?Fx-M$zIn7aOf&P*-iSUPu4lp(h zPlEM`m4^!KwVr+nyxEIcN!Icz_d4}h9o&ZtG8gjVwQcl1qo`uy+T|LV44`GzSIF?S z9PUaBr)!I}7+l;zK{FE8MW8RHpJ*PtCiWTh)4uTg1DSJ0-CADsUGNNQnCP~869`a3 zAlYIrv$@H&%Ua2e2QZv>bbh?zNvA5H6;Of=;rUw^&X2%32GZ|vin&%*K9tv*Zd@l$ zh^d6w^`(H8f)t*IZF(T&`lJ0*10YH^0;8U!RThJ!CXoh_+%iBaBEdBnf_OQM#6&hO zn6e(i51pr1#Kl__3CLOe}Jtxc)5x*P~E+kzJ^GQ2pHvta(j|ltzkc9UT|@;WKUfC z4_{_>^!5Qa`puttVTs4)l!XzXZ1O^nFw}V$dWth?536=9H(O%vDBe?J}DIUPjWMxo7rJ__&@9(YYjW6v2bZ&EPO4*y4KcMFJx|?n7 zj#%DVfha&0t(x*PP+|9SGjzI*Mr~5tUc|c9vPAKTVu^*YgcXu{QdINGV+>UHlnaqZ z0_2BWL`$?lH3>RR7Yy^1wt8TL$;K6>bdN811F^=Hi-GEm4m6Q>Frsoc zDY@5~hVO-I17Cg2p@FgHlf@z|_@=4TrJ07BB$9d(Pv|1aLG#=vFkBQuf5Pj9 zd8@JsJ-VxbatKKVW6st+BPT^fWaS6lRs~g66#g9>AfUmy;8Vx+GU!W;8*xdhIp#M8 ztmx}N3Paybxf<#R)3KfMus$2K_MlPVY5iLnPQ;E7+){oQP|re)HDG@uIo%F^5EcWE z0HfEMTWnD~>)t!N*7$`2_R7Vw=4Iy6U{8Zrsf~E8MFUQg#A{<`>Ud~+Pl`Mmejq0| z%gr{Jn!UKFOeX9J!{`1^NRTbUVwRhX`;2*c&i-Wpj5{LqQHrf;KL~5fU8B8GJ1ARh zO_tdN{;O2&C>86_F4!%$yOVFdmHV*_WS|naJKZmSwf+ydE1z~0zxJGepJ+QgIcg+T z1Td*PqFKz~x3y)>G~D$G_jr?Tx-cazf4EZTY+%@O%X}$93&F~k#awpToE2UKw4rt! zCYY)bblJY!r4fr6aXfNIYkX$VTo{M+FV~ z;4{`qZcJxO(bXPm*2=D6i)5f0UY#gWu6awXRU&6V#i=?Ll`iJS`>ps|m?`gg0gf@m zf7&;_lqz2&ol_8?eoQHO80BbvR;4Ph!y6@`ZOX~=7xCmDWV_h&!CHVag7mmTxui;E z9!n7bNN+G9TWvEt!|&c&d1ImI?4;Mw#G1@avehXO6-!m~eN>+85N5!VV@7gR2+XItMBW7#%#L2gMp9G&OYT@-D>=YG zi&A9-sC5YQq2}(cOhyRXd#7553W7GR#5Bpl^Z#b*X5)2jC$leswNj2SO^~cF)d3?* zfJ#4Bnq}-O%k#me*;&1fYDyc}lA)jyivbO7z-g6BS<^LJYvo8fi(;YnZ zkxa!g#e&sd!VyNDA%r($jT}HI?%T>3*M!)nD zR>wXX20w=ud#}_pxf=Vt530SRMk=F!psi|Bbcsn-ib}o2qPH>Nf%{g`hGT+={st!? z^_w~wj?y8HY?^<|(~9s`d8B}eHVCIElu9K1v3FXG@l$uJ$%3t}P`s(|W4xB|-o2|e z3iFW!eiXhkgp1ji@%0=>zT>p&t}BOnES|TbQq*wnsP}3DomC_i9kf{qf3d;^L#Pp<8@@Oq&Hs4wEUQ8MG$+`GN-OBus0Ui!K9wuXjmn7K7yncANyRTeR>}r0kRS zDW=iIyo$QQI4N~C6F16ms&3hCWexQ4y2%bg-V&lbLUQdxj~1TC8d}bi;mX{{?{hfL z11<_VoGF9O)8V)u_tWn1hqql1XbaCrIqbeT*}v>qc$vU)CykHtkuUucbzFW`?fbaU zfS(p2FN}^$zi`YaWX2p5zMX5}5sUxhYvqd^T3^{@HFFu*H4`y7j<8M^HEmSjM^Tkd zCO2(V;zL&~@^F#cUPN0_sr;9a~mO1M&R*DtgV1~4x*HX2q?C-IWn@vdI- zu3V974w=?af28m0!7*17#%~%+X)|>Vx3908O`HFS-F?$Ke(rkiwJ>pTq2o-Fp9UIx z4mDZQgciveLl(&!LzXBQbMi%FO$sZOA0KdVWN%!Ui%T`LUdT&Trl9cgG-E|(>s{aZ z=|`}p$jdga;Jm3aZ{BK9*EKI$f`t|};imVXZDo1Mg~rTG%#ndv=;oCI)|S@7(F$oz zsc0SKMSB|4#)WtsCN)f_@gUM-bOnBX7rib& z7UhkHZiAhAHTysJv3@fO}kiZTtoJNyjYV)Wy#4t6C7&J1Dk)QNQ9bQ^#P zGy#TrkW;MV-!9d@NV3}W8wd#=S!XRG{zFHtYRLYV$3Z06atxTcP`^nIN`@D_U+yA% zz$tqX_;uDADNBbDFa-wiRGYe*Kll3hVf1I!IwL>cXP2AghQ$)`h#x!Og8+ zFxWLuTn*QKuC>`ir^0@N5tss)0{=1+^7WKMpC(U-`I2=X!d_(X89Zf_O7Fs0b1IFq zLo##0;|13Q0PwAM^j8Plm#sLGyZThX%y0Xx6*8{7YMH#}r_Ua_7I`LCt8zOxa?fM} zHUJ-@t$X7SN*Qxd^MLiMk;069&+WMFZ=a{sE;Y=No<2yt2ElIBrh>M?>?mzE6~)uF zgj9YPn4P8U9O`w4ko97^cr$9(pgr!8@Grx$-h92B#O3M6M?K7S4FrDnC%R^|R2J@w znX;ch-`EmR7uH1TYxa*1Q(39912f`Zl=hUq5%Oc{edloe044UCv(yYZX~;5v5+dv* zF*4ZU=2ulhhg^YFA>p}C(eaP?T1Qv9AlC=geGx)i4e?XaGY0^|j>7!>OT7 z^beUlC&Sm@dq0lD)4c!PP=@V))=P}LwY?p5RyFxwZiF|MjfVu3RFKnjJh|af>O10iBwusVza`m(o0BJ>dZxGIiVs)DOHIpTcx?F0=`Td zxPq^Y4)$Uzsls&0OwAUqTzYUqmYP0)h2|KpEH$~KX6YeSEQuZKt0VBj^qin{3nqd- znv#8TtgW1(NUvqNxXURtoj-pIQH*SU@&2wWAn`jR41D#7Kw za)(%TC>rhoHCd{zjHa*x^Uzoi(V(XRsJTlr5+hlZmdo~Lsj8EM-^ZZ@@#)m8cx*8* z3+j1>hl(fqOBB;-R7_sG<$^*rAtPm@Y2I;EXt!~pbck);GfidHb{!B~RkG02!EzaqY728wncBRWKwxO3p-=ikRe0R|%7qWIJo& zC@6w#Uc=y;aRskY!TS?@Nv$Y#l~rsCfJ~u_(`baj>YiMXXksdTs=S9XDt*fG7)|D+ zZ9MC|GTj`VvU$8m3YjX)WCr9Jl~Y-9A$vJ6lkJ={n=b8$;ss&k8~Y^SKLNsMy!#&N6V1poEq~g*sZjf?4b=ju zr={MUEUT%=D^j=ULa}*5?HX0A^`1VJM8odKmM5>3_L`+wYx}|M8kL}gdM~_e{0~-Q zPW2*^a@$V3afJ0?#Hpg@oRZ z@*$9+LklIEk)Ge%b>OJ6i}T(C#K!~gmGpNY9x7m)ZUgI<3}%+C+7zukE|2THMCIA@ z>L}li=d^!0-aE^GUP9x0a^SG524+lkx-@S%WU2#0{BGs-b#zo6#q+h|jG*Ozezfuj z{k5r0N7w7w{LX)mEF+>+yP@I!d9ON38+sfZ9E_m+z{!3@6Cbc?;}LoqYqXGNEm>5U zA`tCr7Bo#;tLQ5zw!QS=;Zv&St!DnC%oCU8N?&y3)ujXd2$Vd~2833&)LMVp@R8pjrpbP`naWsXzg?x6z22)mwb7}olhN}9 z_28KO*Y|)aZn~WzM3(iik=th3ZJkzCSB$#3PXyQ=dmfO#{CvHAd>7EEQ zIkwGL-VQ;os|9c+Lc50qyINkdmhV1ubDrWO1?w&Iw+#nTv)THw5j1sjABT=cx5?&T zt!Ca_HgU3BD{C*k=@Bj0>`B3NM6&{7HOm{8fio3Lpd1B?0Or$V$N*>#6Yk{L3(6?U`8TAO05*T0dQ4%{SeBqSb=Ev2HU-2#X^~c1XAJtG|Hlhsv zR&QZN1YM@te#)wg3b_(U@Kxk_DXX?;ht93^65V=&@6DjVG#EbJXz8oeV-?Pm%S+#@ zV{xXL9l{IEeTB!KHeX1?NhwW~PI?|UG@3z9!W#T@Gz%~laokgRLys$$d$vW63M)bE zgCDbx_4F}t!GZ`WzJ|47LIU&0tXn-eKO-S3q9^`%zYXpPfF4EKG<<$EkV#9HwjfQ@ z1j}(k{x0T0{#6-J`KVq$V@_poWWo&UqlrODlVNDrQm?Qsf7aA?-aNSAbz6;T>Hmfl zgV+I3gF_IdN!>~67eP7?LZMx+ z(`5t>(hUHK9Dry}>b9M)z>odrvAs;S2(kwqVr`SdkQx%_UM1wRU+sIo+WPg6(VdX( z%+7u@7<1E(X|0RJ+qB!y$Q{v1kh7IDzR@Mxn*|bbTJlBWJ-H{SO11zW)s;E!z+L({ zPzihV(ZL162;E^asg6t)iv=Y8EPj9kRNlA#6_-kOjELS95@&W5kigtURYmXHYhaQ< z!I@AkF4GQGsud9$MftlS3W)adj}^m;2;zl`FmVxq(u*z8h9E2|1PTRVq&Z=bA51?d z$SEP=j~n$6jcwAq$c_d5Y8>{ZDI!(g;DP+_nM_2UV1S7F*F^||sEAnun>1_x5pw&t zcH5KBpDJQ(@F_ACQ|D@iUpEdKBSXZ-bpyUfgFrMy7@%y>G#;8Dwx(qKypf2cr;0z9 z^#%-lM4e)UJ28L(Z4Y2SCdf;Mo)mAKAXN$F7{U`|@_Cz+J^sB22~meHy-=tCSE~&G z<4oh<7aGtHfyk+6IotW6a&(_MOfYCgDPMpGAv))koGDV1Y@DICyCQ*YKdlG_0- zQkGkyV2^4rFUaR{MCI$?={lXo#DSVbxwGZgb&&?+WlArGm`E+rPq!@uYyq>UM}bN0 zd8Td6IW<^b!ByFBrZd!X%tfFcrWHYITf&q^+<_L#OMi>0CRV!ry5vSUX*n_fO-%0z zE<^OH^C7+#fj7gqsddduJ?q@2=--=hfOhF=b*yEdq^MTD6;B!qKX%BAFyI() zLgN#TXj#Ph6b#pqmKau(4{M#p4*0W%-JYVIB>{O}nk_m8;(3k`uPO?yGPE2jgwaPD;iyD3dez={ZJ{wUxSHWgcaqA6o9Na<@U^J*FT?1FpIbTmJK>Dzjz$E) zKrZPm5Aa9#>CC8!+;*=<8!H;r!wl070CJqaxvyDm?hh`w^Tdt>BFPX@5B0+$u*v-c zLG~b>kQhp&t4A;ZlZ&ZZ3@W0yay=ws*!va%ton14OT=21VV-sgn}4(oyYwR~|Fq;; zG(O${hMJT=gRq8;pzQfOBpJ&6KEZIv{l@tD?9gt3>VY0VZYWJXyHCW@wmimf zR#0`oGlE3D8mK)ouU@IK`sybAu^mMFz%=29`fBPi7Uk`urGO|CE@GfYMZiwb$Wfj| zRFgAk0Ma)LPKycWqB9t2B;c7EBrTPZ5@*~SFl@?*k_wUppq7MsQ=WV9PgB-mzphKt zgh#ZFDGNZ6Ue^-hy(1~P!-Xzo9$sDLx6AI)g~nFYm}vwza^eRO*XQ zdzczdlK>$Ng7~|A$a1vhT?5R%6!4&-=@72|U;qV)sJsrzmch-M3XE{HFCoubOY=5b zal=@;<&bSGc7~(vKqrM|tR0uzw#nBCgH>H|?Y9`wAq29hItw3wpZa>pWD)2j$AK4O zdXdp@lT_;w0}(2v(Rs3UdbR>R0>7IWDmq8j5}~4|?G5P?h*ZrRo4gcY_I-Mc zR8Lf)3(Zo{j(`IR7+K0rr5yu=$ggPFxMz1su5H?^SrO!@Xt%_j9MOAL0~)z#j;x7F zsB}tL4mDR4;8@iMP7quWT@bq88+Lm_Y#sc)edEy>1}i_Cx|r9UP-hp)y+@VqFHcz) zc>f2Y4?zD1LYFq?)nM_#hFB$9{GZA5hhs?iD}YP`Q%;8cnGk@;9@JiEroY(>pNV;a zUkc|<#r(tpXiR-5&uK-^>%?kmd?hTl`1Pg5VjC;xQM?oPhHugu?L&2}RPCPgmYUWQ zQlZ|o$IMph4qA2j3U82Salhp6cZBEqYGj#L1G>PwFp82h3$9(!BvZ!+=kT=tnEc}) zD}N>f;VvVYcXE94#4=#P76pLj16nr{lmLrZm`tI=O-X|R4HrbbHI6c6sPR}U z=rL;nful`}z%)TMi-!*|?$#91bW$LtWzuMp6jx3-Kq2QxgB5C0?J8m}KsV2G3UPcP z_8ONJw=2J7TY^_}PVlmkYSs^N3-cc8!{L^bnwL?Z8n(2cv`Mt+=t{tO&&Wc&NLIU=YIB7`12s4BKy#{V^!U!7anF}CtMVby(}ka8*@%r z9XK0@2fm8Z!QAv{zI0WyvrSXMOQIxtQdgdAydRnh}R$$iU4ayfo}>(6!17o<~`>%$CwxHtqEzTwQzRU-L0n?iEwV>fsZ{-aQ)r(D}ZVi z{j6K9qKR}7Zv12AQifDbX6x4eCzxx$m|Mbc00iVhjb^)<|Bj@J9`G7T5+IB?VQ2_p zZx6kSyi|;=y^V^yoItc$l6=rNv|6(iQSi!N*9J2?&7%`@`ul^@`qX9VTS}vwRZF(; zsly`suk1ztO3Tdl{ClhHMSf2_S&?E6G43h$F@HwIp!c7|*6NIgg5NFWKMSqZ9$Xdc z$OnO85O)8xqAuyh)&`Ll4vlGp@%v!fA#xvTbnpM2aY!MIXBM{<=3~=iC;cb^*bUzu zjAoxs2NT}`^bxv_>)FSE0an$oO3&MOvpGE=x(8-&rcP5E>~oStviBP*lfh$zLDcGQ zo7&W^D!vRTrt-a>9uww$(2U%AF9F5Kb#7MBdFGYYodg-7-QanwpxoJeDy)Pb7vkEy zt?0{G?9VSP$1{&9Mtdt;lkxhpFc>?IzI0ZP);0XV0q_1&`%UnJ6`!VVKap)2vt)=UAv$Q9dw1Thyr4IjtegK9XCg z`>)cf9;hd6`4(uPFN7^ib0BnEil@&dr1v-#--xi9*qA$5JzgZ2!I`02uT;#jJx-se z;6_p^gTsqZzBcp|OyfT0OJYfLovc=AN;D_Gu?q~e)8_f*GT5k6Xe%j?d z&_)f-Z2V%xy_f z?ayiUI4<~(dYJBO3WBuSL{nDKj1q5eKOADnUI>q3rTht%oWw{nGGvO`zzOue7ns4S zyKha4?w$>!B_o-^f5wCvrPk|;R$qaKrcJKvgw)=0Rj$yf6GAP&A6|+AO^PoGop*o# zWT%9k)sIX<5($vx!d8wajBsCX>jowR_piimFM}`u8$q_6TtzTIQxY-Segd)f2fEe_ zHRvT;8M)o{{RDqb4XbJoK#_-fMO3GKwkF=8zWnzbpyed-7HH3BJ6}fHClm0?hB{p^ z*rg;2$j|osB>B6#?`^hHDOnV+l#7i>tLkwB3%!zrP$b_hl|?nvpFBX|MUpN9i-B2f z`}oqh@S{5MwSU5+)RWG#m@wyZnQMsqPHQ28K-|_fzIGu}zlYs(2KM}fUmHJia}*rh zo7AE`CY4Ljc;7gme3CYhoe%!=92Jzs_^Y~#@k0Aii&;_FQTVhh8zI84< zy0yMkj&c5m(}`I*BdC9N#hF)EBrg-+yX9)m&4v#@1}F1JB>tN&e@>a^qhfn+GamJ( zS(^{_MDPiE9(0xY@xgv{0&?$_2g&sNvo!bc){`D9)b-@D9&AJqfc%)Sgnh@QLvi7p zuaEP0sV1QJuNi|NID2jJ7dVYh6n7v`x=U2qU!or7-3#+=Oqtsmg5YyQH7VMzxx9F~ z$ov}^IQ6z^xF_Zy!mAhjqMe^s0{YE8^DpkfyWw|+oqgBRHnas8Xc)7;Ji}2dbk1uC zGV8^ea*d&Rs#mT<{~x`SDv=9Rt&s3%jTU?(Pv|RbeCEQv;$a*bL!Q%MVuKiXH-q5P zGmbj*$c68m;Fn-V#_l^6W7C$&P@qibiNRQdK5 zH^#2aJ;e@Cry{218#J%Lb3EnjPS{ACD>rDV($Uoeo~?j~bsgu}^UdM#Zb3Dki6P!~ z=7+Z-sf*zuDEEC1&I4Tp@!HklgQ%mmL`(=*=+HxRri!!^xjopqsHTv@*(mCXfsgpY z3gV&b;q|=fB8Nv@v)J%$sOdJR`BM-A%I)~eL1pw8zq|YF!+je2CHc8Yw~7^9G1l~m zL?xC5-fFIU&w$ErOC_R&z{Ku8Q4Yv+CHUHMxl!F7WeF4y0+RX0?CqwlKUlAq z-+<&htvP|DP4wPYoUmpsK$rSH_XIDzGf{f{EvVfA+D=!HY@Go7)hlm%ttA)1MJk?q+y^iW{%LF5iVOm*=V`M;X9^JOC z6fj*xf8ndRH6CdyA67cP+)oEI)q5ir z{(SLa$N`EV3U6?SOx3_@u}(96KveuHXd)xAU2@0GJ76;XUY%-FUt9QU0MeUR@WC`d z(}PO;t;-WKI|_W3zNZ&9`t$+O6Sgbu88Q@&^x&Qo8}RxJY3g|gkLZO8sJMe){*BGK z?(0tK&n|yQj0YnHkmDPjMrW#YRmw>Dt{#D}JoiLIc?P`uOmQa(hhJMLArA%57=h2q z^B8FN5YZd7n~La#(_c&Q`p4vx{gX@ZakJkA{ogRZ z=&cz{AJSZ-vE zy0@XCuXi*W>p9`eXn3tYgFEywdzo9lG<35}w5OyIDZ@X@?)av6TXoe?1$}60YWf;2 zPmL6rY{j%#9R6B4!%I$eoWzb1cI;pt@a&JCyDGC-$wG{SDOz?&P0m8qr!jHDfi!)M zzsa^VU@;CL=@;IiatL2RLdM<{oGv_q2h1Ukg>x{!-iLqviNtq!1%?zSECB@(0j#3o zRJ;{u$TOW4Eqx_sO(<1U_%@w$&d6Q0Jar-SYzI5nPb3AQa!}SSm4bbexJNtUl;p!$ z@;)@Cr&pbp0j=O@^0&a#WImfgvMARo=xV2KPd^&HYEjx@`6MwC2EN7Qljj-Sp&D|T z8>&58*_*o?i!>YOUS5sS0h71aQU@)W*GvBi(g=U}#Kq=5x*mi4&74-&(A? zB!IMQ44LHN_Q;{n-%G>gXGC*=SiMka$H}?Ky6O@`+xIcncTOK<8pUi>#p$sWtlqM) z)BeVQ(Q9>o-p+eXn4?z188XV3*Wzlc3rrkN#O$2xEZ;0e(Z}vUJrMYTE}z;FpBdNk zeTDG8yjF^(`zc2Gl>N2Z!`op`7IX0~Or7*(c6K~@J)4~G0`qYcumJ)xQ5_PGnW2H; z32T+K3ZbLw@s7Z|t-iW#w+yFeJ#)QwX(EuCe^{A7jn%t`yjj=kKvuQt2QqECVyD(c zNMh^C(3hw_vMsL~#c3}8sBS2q63*_Dja&`yvgdS_BpxrE+Ug?BE%Vjkl&f=rF@V&w!bD$FZ_Z2@7bW#Z<8{$ z#3m#BckLD(>3^FI{CWpYPbiOSYcUsQvvKWDwK7A!88w z!(?LZ<;)x-fcBVk<*T7Z9`^I@+}hj9Epc=~(9sUV%#;!pub)FgU$e=g{WOd63k?#v9_&n>6($#G`N9VntF;Ljbq6RRtZ_y@ zR1M*=dA5yE3f3;rT8+-VM=>W7+B=ncgW6)L+FyjdvuL9nQHcQmNcj?NS z%GK#k8=Sm*dUEH=<;m_}V%64`&6PP$tfZ`=GvM(C-H|VmfA3>ZF%Pj(GPxs`upK;A z0%soK0jQq>JWC4ZIVOB3MZ?r#P#^Ow0qFpyRHm-6VPYMKRFED}+f+Dk#gc=Ki7_Nq zi|a6yvS`4vH%N=1JVwlyFkvV|DaP`8dnRD!nH%1(2dz*1*n8jSLg$4-uCUQq|HFQA z1siLfv!ymI5N2Ub34*rivcPLsn_nYQP8!u`)Ei$`R{G%YY>aXgizD z9Y)g$;q2sQ8{ip^Dyd|d(r>fV*jj*Hu!fz$z)00j?dkF?kswl&PLv-lPsgCnumH;g z6>^zCmht3eL9EBzzr(}|Mb}7@o;ulQHW)yhf7^7T1->Z8MA9Ni)Z%2&yL5b;8C72G zVrP(&v%>v5N;XiOL=_7b#iQ%=FDp^1)2&Tu)!~bfdM@uw5DMA)@i@OJgG?1Mii)cQ z>DYm>wfan+`UBGCbxq>FSw`Bc8+ndwVH%d%;TavXWuuC+e$p0)v=&jP5+bb=zizi& zq-ks|qPVfs`}uuh*XuY~#-#N1G&?}C zGc)6H1=w`;dHcE+)GAU-3U`A4xcs&>FsH=cF73LlQg*7Wqkn@h%>!S3I>_rB^4&Q@Y=&SJ7^o2?aP zRZdCf;8VJNJH~qS-a2vZl}#RUTDNu!#n&8Jv=E9`yauSsGD-UwxjQh^V@#w3yy3E~ zLdH9rL9n%zyA5l~ec0dw!SJWYUkSB%b5<&E?x0@}&+@zWqHkVj?-dkpkxiRi9nVcO>Y#xMUcr}z#|G7C89?rVJa+r>BM zWPchH$C~ZAvaH7prHRZCspNYbl zAi~44B|95io=0s8PtvuMWaH1wQkGPyqe{I~GnoLREtzBzQYl6juzp*Iw5?Ew{b5;| ztg6Tyc9V?(048+*sYB3b@(%n)#lV==bOYwa33dhO}E zp}Gfp!xz15r>3y{&_Z;zJl}Cd6aG2H{U$ZH3n+FBnUyos>xyJnlFChO0FW~>5y%T-`kN^6J z|2inx*e!}7=x(HXO2Ll)&FRH*P0O1NJnmk3x0mI)ChXH}FT&MkA^=-_xg4{s9# zf|e^OOul>vdLzZ86`lq5 zm^UHf`_dpnDXD8?=u2!B+PMLzbxK_&0e)#X>f^oy+0itqO9_*H7a^^=LY6 z727CL1lM$;E06G)vFxygu?){o=KC1kY$_59C7}F-ECUk(0=hGCAa^1dnTUx`zOlmu z`rH)C#f;fyowLdiZLnIZrkJQ;w8|u=`EyEh<_n%S3e<}@*9mX35z@?Y?=wtGmZN*U z!w*)(V+Floc14n2hfeGjhOJwLmfjvuTStMu`}TZ*K=C_Yl5QqkY`!asdpiWj-9tHc zq3-5tLd9lMLA1l47-f=STQU4HPF2nmw^u>&He(*|1|NR#_CqK@91L4L7>4kUi2oWH>c|R**z~umTgaA8Pawr>2yMU)l2T)^aaz4X z@;&xT#eS40?eE0Qfc+B|QUjFkF-|=#Xn`acb{H0{L}rxxMyh3bOA*Ebo2W7=As=r{ z{;!FKMEpoj;M7f3(_UR3AzqMbIzP&TE>rl@Q|iIN`!cwku?0zd1knNt_Jabj2^pSv zIv*2stV6stk{o1CzYJCdPkgXPC9thAy{dYE;!sx-tpZY;mCkrQQ*)s#xZ}Y zh4Qi?LJr3z5YN*rX{o^vo1ZKKLP@2wzb2AtiAv&~)YStc=zBf{zsQ3TreN>7d5ha- zt*5)lqr2GuQZ}{t{`Huk?h)gCj^!DYJ0#lj!GR~ z>hv??e7#Ze5%D~fbfuW(L!V4lH+=ck zAv*=|dMi9t1kC5+p;#6eS6H-u7INiE>Ekiuj5vD~ulEI6E_~v(H@$a1X0|1Le%Ay` zk(cJ5!&r$p+>DY&HggOyz+`-wH&Zu7Ls+*X;|HXkVPRQ!qMvPDZ#mD#@}g9C&RK2v2S?%rse)(%e1LB(y@ZE*sHI430!5F{ zYs*|LhP6TANr}9E89ri%YW>rdq&|rI(WI{Q;chtGmd5zi^5N`kn&G^Uy8GZ(9|i6s z_5LHtLWD?0U{JZgv#dpW+$K-w2G2jI?6l;~$(~)Xiy1sb)5b@*c53%QorM)`=-CD} zG}l)SxnaH~qB;(C1eUD=wgH=f{`9ukhfZYfa`A>}Y ztU!D1_@wM?DDF}z!&dd#$X5ck)H(GYGLQsM;pS~(7~b^-Zwxq&oIqZ%ha2QWq^{F% z{0}_4;98EZb&gGnm7Gn3s;0%=>}g}4DuqUotthEZZDE_vQYw5eD;ZiS7sP_KzO}>P z?6TY{aLkqt^2|r#Uo2V8#OL1sK9T&-y&OZJDAwJVy*L zfM^!r5;YW&o^iD6`7)C{328+vpq*ydhtWEkay(4g>o$?wKOe*F^B}IhL-lNWb9{hT zKmK|&>*(@G*Y~V2goWBO=6oXaHIhiSe?c!)_skY!^|Q$Cr9hx=Iu0KZ%Zqg*C*3Iy z@hJ27+oae-zT>4sp&g`dK<@5nQ^lLvGhz+;LA={m@_R)h|&Hndtj6n{vKyL zZzZ+o#4U)@-E_z~s}z z`(>e&-tG3{@!8eG$?eMrEZ25#t27h8_NIn^Yj*)WZTl09UvK9V13F8HFT)dpfd8EcQXF)O%;?K2!@$Bg!IG*SX)Mt^Ln_v)(h(&Q;@Syag9s0l zQmIvM9MM8s#&D70Y_;hIYzVk=9BQ{N{T?+8-V+0ZhbWh?z!tD`k!iViL>!^tD9#ry z@9*N)D$LyXcK(CA$e`xApCEhR^rs&QB38NuwU61*=;v~#{H5`m&QUj^{ z)(w%GU9?}O_Cuv5Y`RLk8JI{w8zthZ6!Co!qx`mqruC|F(mjmOJ8Rm(tp9#G9#-`! zx~P@%vf=5gRCAn5PuYq@vSo4H8xxaQNZ64?f!jm$8`fpNS#b_W;h0}?oKzrT-fk{& z5p>7z+RK9uM?JWNZV}B0g~Vb%;^`XT`z~Ii(&{7Z7pzw7s1-u|G+excO<>q|$4y&*-UR2e6Fm8>1=AXQ4O!ZLW8KV9nCTT?IUhae~XPF ztkkA5#aTh=c-ZP2viSSlS>Be>C387u83JwETlaoQ(|W?y-Iv3fw|H3%7c>6AJw)451db}n zUC+$BJaGnwOD`tw+4Ixmf@uPOFHftgeG;;FaT(!6@9>|?)ZP=N$Vff&l&OUhKW;ho zgL?-(ciO!VBD#XTz@Q(SL7W$9^x4!u2k)*^i?*>1-{`ErqppY;K%lLji=_}6WWb7) zG(l+wZVkok7`n`HHVt6IEXxsD1Ihp+#BfX22@c3SHKcL;5w}30kD14?jzQE2n)?`5 z&D_UTqQE7l-*bnh9k2w1ljUz%DU=hdJ5INt-G$qV#D##9J>+ckp6iH)LodlBOPB-; z-#>$DgeF5+XM+G?VE1=9DhR^~+S42MbwNhU(T!0bskzHk#+e?4ULc;cn@}ZdL6h}x z8pMK0B!)2$7x$f$v;)!=rIa&wdGRzHQNr;thhi~06Z0_vJf6~?yPdFK9q6ZvT(B8H zrI;?_8&y#t3|kq898hjef9(*xFR~N_rS#`?o(50#A3qf%9x@0^ZJGnqyrNvDpuY;|iJ;#ZjA_#2E zK!4V4du87rMp2fBHAv_60>XN*7P?)rxNel}(_;_JYaBD&b0aYJJ*~Vz$aJu7Lk8DC zCXz-=n_QAWS-dl1vntZW?;MdZObq_0d1@ZNrEJ;}6sL0xOl+s1&pO*G-F3s)t>z|G zh?`ae=Z1Z<7HT9|%k)+0XA=H=!g3YsIG|#VM3*bRp{)_LI2L4ZRyk6d0@{cgNwx`} zT4SRqbTM`Og&?A0IQ_o zFH_A9Gs1(+O)MYU{ZD;y&YHSSL10(!YKP)wq`XRVa4cz%i~JdAY+ zCVKx-b+t=f)NFoHKnxj*PPU>~IHQs&{#qmh30_0!Mscp}i21V3+f6pbb`@BbrB97I z#*6@BwYdSdRL$IK9LayKh6bXj6tds6z3PU-f4_>m8Quxq4*Jp>j1t$us1>7HvlV7H>R;Di0XQKb$Z+r>U?e3$p?y$D z$nc&AIA6f7e(2a+1xON(9Vau_E0PK*%kIvVmmJS)&X>`w*>=0~q^}5S!a26o2%$)>h!~pn7KwZt>5USbjV9@Nx;fYJrpZOhg%)8#UuqON1@&U_ zDqX=;MGWu43NI^^Au1;@-TVy!Ph#!*bJE$2-?J z+?tMV0+@rMD=^8^Yp{;y+304BNG(O}Q%bskt*VW)i?P^Pk1dtV7)TmUAO4wrTSJmW z?Zl05j!dhStWZgu$Bd^vXPU68dy_?TJX|I-{CR7;qR;k}cYff%&kIGpm3oafN^H@e$gTO8LN1rS<}@Avvp>_ol#{-u z=H+otC89H>y_)iIXhUStbMD(0s|#$(I(-s!sJyl!jy8v zHxka|6>3z!)SIdBc-e+6J=w&GAri8INa=pIAcbGT2IsdfO31Lhw|Juxte0eox8#vq zH&!YO_T2)u$`V&Rw=Pi`>(-X$CW$fOCp@-9_O))rPiQ*&b7GVre<)Z2N!4A#Bfr|Z z>62-EFd$*pSB`GrTQ~4^-?w*q@AhURCt__cmy}TSaj{!fOhhs(cxY9G?V_c>h_Y? z?DRA~m8@auCb#XGysI0^o)3>h;710=xjfBZ*T9F@Odfh<=M781GN)_NM|-8=q$53X zKo%lfNab3nLQi^T=h2Q#LCM&)n@`dhn24Ik5P9Nu{)^?_d$+|f^QcB9|4OVRr?+{< zW?wFJ^~9YOCO-$wE_`Bv8lcd*xoAtQMlatRu}wN}-Ueojk)-LnuRJN_ztC;+>Fff?Ct<^n@=&f)l&_{Br zT0+#Alt4xWI1HDYboK$<^(|On%XaJYJxr$LIwMAR$N{h=Ne)D}K3Y8Xe*w%Kt@fj= zyZ1H*1Ew`41nLEOEL2MAvPA_+13ZMw(v}5en8JN_Q7>_pyCWWk3{yt0vkfGQq85Wd zj%4Pu%F!%-vEH9z?h+W}l)^c6O2p?lEa)KV1^b7?CK)KkrnJHXi!Nez-!(&WcK|9& zvOaIG=$wIH*(h=V`J)PZY*8lxj7Ilw7YO#a>`Fg8a?nId-MOSwhl;DNf6|x@z5QV} zB2(*gFvqT%fUPkXMdkB8V;y8YKsjJ0^BfRvhJ~Nse0py2AB&apa5OSCaYO@st z4Mz{>p4>wW0ieif_4$3>FP|;t{!@*lX%Z~T zRU&xLv%UG&dl>^d(@&G_Pjf?LFi$X#8S@k+8<&QH3{hScYEYm3e;-PFEV(ZI184PN z?dP$7K~_3A4INTig8UMV!tG;3jYzkl;`OtNv6?Rygi(=9)a)Kc?S0Ei?3@GrWwJK9Oa0fvPRcQhMQhJZS5xGOG%& zeLCFFb{(mgu=Qw?}Kyqc}JwgQ}4Bo^OZkD%e zQg^Xq)S#YrQl(}q`49>Pua)SB^uP1ie-JcPTRI-;o&a&;oeULmc^p?ZPCeCbf4^1HqXcA35+F z=1)3oEq_!QQ?ygnMiEyvbgWvsEHJmu1PlmSVG_)dFX}MAhZ*Xl^k?qvfh4%sb2GcY zbKqyDcDXIpN2^`Z{$1HjQVMGe=y&YZ?mYG`4qKGx)MmC`rKNc;O6_@9UJVgHfq9V? z^3wv;u^)hA?Cq9g|F@#InN->1Y~z|Z<}}l_74)T-e{YTKPr~#Z}3YE z+Us+j>J2DLU%eadFf59jg1+?OMA!oA7{pC+aOGJN3ls?!XjyFAm&Rpj>qLqVK;nvi z+x0D}!{qFdIAu43t%7{}M#eB&NScKwl{9ue>^H@|*E{M3bc^7=HE1h4emiSiP zPPlF|3+RqI6CbSSb4m4#FBZ1h`7HaQ@Bgsb2YgTDM){5U9Q`$k{=X_?{%hchOTU$pv^h4)eP>Yv3?SM$Hbe~-NiNst3pep`BvcL84p^OF zf|l`vEw>%qST4N&@kV3_V~YJZP_Hg;r+esVmKcEDWEs4qGSG^H~-m;wk}357w}745Bux(yB?+#8WTqiJFFVL~;1v^EL@Kp$gf+}A{ET4NZq zExaa)ymVRC4SI#aK<)knUI7}|Di(26p+T8EmmWxWLxG_%pLCIcVZhL4252iXaKrTL5yPvjicj#O&cJ{!^jlCm?8oS-4r=^R%tJ`-d z-){f@B{O&KE{`q17?pg?9!Zov@wfoD#Zh zi7J#A?*NKqK(-|&A*#rv zC)Xpf{GYBl^B}K~@!}gzv{t)A-1lXRzUMiwYjNudVz)T7SF3WNow{~zgCYaPiXm~7 znW^p}ulwfLNbbPe?dYza2T$j>er_k9Ui_$SkCQ8CZXvJv69O1o0PF=M%PE6HaCr^PsJuhs@C4-@t}_FKPK+uAx-0Pu#NqPg!YN8eeX=Ar zeNgkA==eiOn?D%hj#p30z!Bt#&l8 zbVR1X1weqj%6zjrYkVzr>TFE8O#3>e)`m%cl3UQIXj{^uJul63RF;Vq#Dl@VDj+?8 zFpZjZ?{qGz#ZTw2lSzU3-R+f(@^?r2JMpDWQh_v(X)T=Af#VYr1c&2g%ge9GS8%S&Lw*BFD>?Ddh zS^`Z5J<`w}ZMJ<0c6$#ztd-}rgrUw^Z)8u zN*y`R@O8+#q~{BjO9)JEdd8^7E?l|F=I+ebosN%I-f~Fvh!ltIfn0Ar^8(m0JZCN? zv(nDJa4IG8C(4f|g z)r8Ma<;JaHU7LP;eXjsmsU4g0`RDQ~kkOnnA}gH9t-~5Pri-+C;z#AZILu|FdNx~Z zZ5|Sef#{VD&ddzb?wl*jXUK344NR?7_(@X(PMRA=3D`p!IZ;7 zN&?PoA;m*jkyZh@ubMY1l;M!3T%Q|iZNM#w+pdbi?}UA>!RDYmeUTtV^$GONQx;@% zx?u)ye;#=2v(CTMtSFqFUz(Ymhg*)%yovud8N1;aLVB?sm;>`l zrx{AixxqUP3;BN+6sqC_HcOtEdg8UzKb>HN`7Luu_dwb%E~f+`k3pK>=vw$9g7tx$40!hm;IcbO&Q%$kwQ zy{$_IRtM{i3+*}7M?;Us?xxrV-`QoVUFt%yFW5*GwJ*86u2)o$47qQT<3(tT`*?8N z7a6&bdp4IVMF!a+$=LFvNN;(N_;LFZH@VH*-Q;&P5t(_RNt({gX8bW_UPN!EV)maH z=z>6#e#oD%^a&2QkJqB%dWJf-+&kPy71&VHD%*)kGVCf%kUgKa{e#Pc!K}VEcUmD> zWW0fCX>{?g)sCx%padr;mE!7yF@<@jw5Q_Cs7U1UV>)utue;z4`~TY^2zBi`@cNZj z5Ci}Ks{fk~!GB5XFHz8)h$D{byVbaZ|I*I4MM%uXuv~%QNv9>@7aV*Mg>0R=2CSk@ z7_ye#)83w$ot}M$>RGWy#L*t%=LkT8(1$O83K)+H6pt(p2PqLIQAX7IwSAGk#VzSq zKJYQO@%8TQHha_E{Jmt$^STTFA{N(M@%nWf3tH6M7Y=RjR|udcM4(TzClHX=JZuFG zN7G1m*8u`f+DIp?5E2MAsY*~b&;|z1x>*^Z1>6c}3yfFbp3R|hFfeK}FbIXiBG}w# z(;psmr~~1jbp5h^s_KCm%AWmr=)vx)y#n7|gWR!|Unnl#cKU%LA-|ia=3pOUfg;n~ z@a2F*x^BHl!SmCT`>tqCLCrA~j&J#4j}^l*zr%!$ienm%>@iW=Om9(Vj}f0RyO+xG zOkv?kdI)|Cz~r}bZPV8UrYWEr$PCDzz+myuSeHEJZ{R4IIpioBPAWm8d+Q&CiNEKK zNTgYAM_sbxIkmTKP>t!q-}#IvURT@oPr{H7tSy?sZ(PO{U0~b~uw$x(TK-xT2{r-= z5=c8iYpiIsn0Ne8NH=|-m}4N&rno5E0oIW``GK$8GI(8)W)mmKw6Bmp_SjW!G zz+iuOhOCN4s&kb;+2b=cDSUPL%DcMlKNc|m3ibCIbY+&J2dcf5%g!9V*}7Ah5<6U@ z^txT6$kK7gmj1jr-YdWF3b%#b&nB=%LeOhrMyS22@E`tdPou}V(U!0}IYV^27vO6e z({-}A5rwX%zx=xQty|Frd9t{obIJU@jz9mSoQp%v4Skl8&npq!J{38$S7;Etq`jlj(6TfBvgcc2Vv4XzsKXvW^$`R=LW!gUnFZEKK%k;3faR3!N9$+qB; z1+sHaMb8Y4+dhP9?8G6dX9{XNqUrrTAVkOlCpaqeT|(#Ista7R6ozWDRS@PQ3lKKW z+cu)S(vQDk+!Oyu>I+iux8o=?>0z#0*Uo2#?hSgPxLemtP1i&0P1nnY;r%tP4%wT+ zPS+gq;Sxxhk=aPEKNM@eH3su5(R0UK8tsCL+5@E21=q|U4&2J2xd@k@Ymjm*bx=oc zhbo1Tpx#Uw*vLeZnesK+CyS~mv1B~X`g6p*3pBWshX6I0LDigbyNTg|hqpuu2b*8> z%2X^|3vvb`Hm+f%m%@1FCQ1{Wjcx3Y&(5Ff*0pkWy{(@XNh+}E7=1MB34y}8eztZ) zRPix`QAw7j@Pf=s9~=&18b{lS?a}ZC$s-}u#*h3;)4vRDW+_iX4+gvLlc>QXHE_wJ zWHQ=fCE8W#zVVnbK{Q9BT4nfL0<&Zpf5=0&6C8!Hy z68NAlkF!QMr7(7pG5c2U2 z{Yjuqk_#zQ^oX!sQfB6G4`srTyx9#;B}&VT%ZGwGo1QP~5dvRN(>831moVK z%K$buItRmMzDnY;mP9Gf=w2WzlYjMo(A27Ftz0$&0OtSCCT|VJW#sMYXdeEcoXx$r zp$Ki*&_;4#z(@)qwJ0l$-T!sIOO+uy`)7PsN7t9|n}VpIJ{)L@4J;bR^} zZ(?)AEbxx{Z;Fft8M@r~0BplTRamu>=q3_xB*He5=W?pC=<@L7KBmZZgnFWgT_oJB zM*;bo5%$5I@WydSA&2k(K+vA z#|y^O1pzwvXM&$ji;a9{F_X#1&mAWbQ`=Z#H>@(X)2Vs7ExUHITUss0YEuJSP5!*O zF0HSWPmurC@c+;gGM`b1rN0_}`g=Sk|6erxH;Yd1zbJy=P`YV#oBte+e*ppqC%!3L zloh#U1L;ygUA;Pe9nGP$kpBPR1J*L{ezzVJ)5x>MSJ;Q`6r z5D?T&`{4&qyl4+{^Mh=NzZLEv8!Y>rr>s;aG(+&M2mSOsYCeDwP{la%3Pt8Tyw#gvOa+i9~GMHHg%)9M&Ow*u*%9{eyj-PLGqGR>;Wr3hCYN zz+te#Dd}xqUORd?>D;m%ifSZIugxJ>Rp9k?dpe(fx}3)3vR2_MfmDNX{Qgiq#5li4 zR1f(MvJTfJvj%4ZZ<*k{wvhb3cFae#g-?oliaElm77;qrrfKZ}RNccJ(MA_&Q-}IG zgQ+p~el_6g%=l}5uu{2!zvJ5#tXIm;zHqTo$Ms=;Xq1Ft7X3~;e|(@5ogT32Qea{ z{9iXjHHjfH%Yx|**W2X9Oh54 z))gl}5+Sbc=orSgp&!bv_@qRt99Uu{MDn&~*s`nR$p~gV{kFG9)p^cs^*^DSx`cOl zWr|G0zfki6z+aFaa4GJtOHUo1@8*@J+_e^3fBLfO%El@BU!B$Jo zvI5&MTq{SlY&CVTUa8?OAC;}$f_yT=$~WF9@kDgl#7FcA6!O&+b~g6_%Y(UBpq|F{ zoj2Tba*&*GzSGbeNY@MOElt`izp)(M7ReJ>p|6FU`CLLjOGuN)UFLL z`uh-sXMrt5@aM*cHLN;#EyS!Ya{-PX^T2;-o^v2S;tW<_Z{aR4^o7;#*mZ7MW~LC5 zB>Y4s;jDLh#Gh^hTbVXTdGw%i_E8jFIPu_nbc1VFV68IoO5;30gP9V+rOlcf)!|`? ztklTMFdfG*c+$-6o>FS9vzRiQc{wnlA5Ld=|157?ZCgtA8|QDwkq^wjd~BJp+35|( z@3VN}Dwg&joLAsB4>7Wv^H$Ra3KsCwoT>uTX@b^dbECuTca8f)dXym<st75Gyr=PBs>v56k0X|SPK(7;Y5rR_!x|JHjKXv?8R~o}R8QWR$tN`uLcBUt${Do zfK4(1YWoiIEW74oV?0BmXf4HAxdbUhUnwLy*t7@){_68|XggpxO-YzhA)J%zs?7jkf9+ z2o%mtp@`A4tDkA+?)SUEu@MCh!~`M zmP85t%m;WMS1CO=EgOenvD!?xRHHW4&T})G%!5ON64w*;CRaD*@K1gxRB<4|@ERh> z?#DlP1cZ1n!5xouM$uS(U!eci`u|My8=&2TaDFHH%fHzF|3(Y`53T<_txwx+j-dJq z*F0%QoSj;L|Mu!@`!q?LcLWeDwGun z#%p$m5fU?aqJ+YFW}&d0J0`zqnfg0FfY5sBl=QHLE}=A^uN7Xwry!8Hua>TFenBO> ze}8#wpMMG>km`=f1D*D-<`Uc4h(y@SGA=a9dhaGsW0n(NoDEULFAhIV zGn|_%7WNDYE-q~KAsZ4j3fPe4QYDN27(U=uZ})k~AiG9}iMYR_Td0uMSZI_mi}JUe z6+@b-1P_ajbgin$-|`rK{g ztG8ewR~}!V8t1Qi3-^NUzvuGjzg2J>)n3QEXRuv9sIg|1ez2wa z8~zwG+ST5zfHk&cu&HZN`B-Y2ZqjZ3?ryZw_}%357~!v{qYWPXfZG;RWLT0=eJ->N ze{-Q4#nPi%@0~SJp>a2JVrP$!GLy-M^S1@}pF3)CgJI4nzYb2?vW#EkJ6dycV zC{pMz@D}Ft?_K_W+MxgWDU$csX2RcIRF=3EvF}0p4(GQH0F1S{eJ^{!503-`A@Pm< zn!~VK%qs_<@-)X#8~7ANAq&>ac;`YHvHx_$o8;-+h1bM++?fLVQ$C6#@veEo&!W`f zw+0`($iV;bxsL@P-j=wh=EIKBmU%b~x00i0=WRL$me1U;B02>nbnP=k5K9 z+r)yCnz7Ho8?uJdyM;M0=0r?V>z9V-akfC~Sd0IO8}^p~efLB2Oz>DEH9}kxAx}zq zRHjMMiZAB7mVy~EW1*svR!?XutAJC&550;Z;=tgMAi}8!QG85uC?>z)WOIly)JQ zg}>1}Q}7(nem+L?%$_X)z^;G*e@8ARJ4O>GF_55tfpzR`^(44k;4jQaKoO@22VEAJ z0RzG~moi&zq$n0`ccy#+ju;mSi2YqbX0&BO( z&=iYK;eMmQ9gwRhsTu_OYE*v-srnLjRCd?bkk_?T*hwFq;At1O9J>t3Va_WDGltiq zF!@~z{SZCBYX-b)2)3SxT@LcZ(A)fd%gN)|5RmBQtUL*#J~aSD$8q-_hv)%=w!XmB*~30-h?z@AKdxjxDJTelNb?uDl$r2t9UzkeiK$V%QbE>tWA;NkEOBHH`; z)$U{{JIWt&1B9pZSBefv(}_-Ya4Wj#aDHT(l|#uuzt;J6ajmXX5HXl=xgmMb?V#J_ zu^zrv&CGegdZ=rc5|SW~>;)NGL$VNe&DP!IJSaPWTl zKu;85uD{WPf(nGPq2^RwZ7^x#z~zsY5IxX1bOy4#F<7=eP#7AF(~Gaz+{8%_FgClP za-r+U1p!^PN)bqbKQ_RD5sHLj*&&xHJyAGk6&|(+ zCo^b;P>Zcp29mp864D=~6hs4>3o45PW)m*;vH6g^t&&@2>k)g#8JNG-(`CK=a2 z`1yKR|Mq!J6?9lTR4`9@jJZ*%Y>8((GVLNhO^KkM8`jwpG3pZ=ZHpi-V$(4PS?4N>ev+fQUdyL01 zw>HnaBVcn=URB1dN&IRbd_ zF8=tX@J|`JgU^fM(XR8_?myQMtY@JUN~yLw?rl@UUtUejWB=kr)n(sTGw}sjU{xe{ z)FXv~o*JkSkMiqyTih|q`nw*U;upr48-`D=Rit=bkEZ+yqC`%Cpf^U4lRAoS!hr;> zv(jgviuCE}`tC&{R_OPg^Z@|2+qZ!>fm>kfbbFQN(sy&7^ngaCN+YNdus&wUg<{`p}Y~rmasIV6KmSiV{6A{KtMm%}C@=s(B{Tp4 z+5e^{I5`^XTbSB7m^%GOOi;Cr-w;FinXSWkcR^}4Nb<3&xB>3Lp|YB6>1T~99M`%6 zWMk}_z2S0rf6mm!UVF|Ii$tlAaQAS}^E}B!A5~{~vjnJ^ZeNSpGvvhlGP37V2XLT{ z79U~P2HtK4r_ba4Mg1|;Of!DQA1zszDlx`5<(#Hf7{Xz#l+PH+r9SW&QAWXHrVtH- znoi4sN?TO7Js;{T|5ppw8=fn$22`|-^6EnzDxxtQ!xNMME)gr-!&=EHJR}?$1VAtL zQ$NayL@}J3SqZI>lmP3lpgfvn5MmxgchUOfFc^hw&97k`vEbOcePwhuGOl17>IS;l zuR6cpwsB-97??d4JSu0U_|k9^t*FHdL@g#arB=#p;{9i^7bVz;Y4v!Iz@~P=Yt_!= zQIe#^SQoujvIsWBT~L_g3mh8(72X4tMnm}F)0al`Id9lLP$LcYoqwDTj@8b0C*?el4%~u@LW1)QY8J5OtKzm2iDpw39Niv_Z_A z_g49);v7opnFctoOUTj&D*zkV^W1U_qUwZMO&}o|Cy`V)?!8f0{F)OlMb2x%l#&XJ zKn@09GMY2V!gvp~`fHT(6f8rvF#Bsp2HLu(LpQ+~E=@7{ScM^Hc?D6-IrBIZa*TEP z)6S9Nqf-LK?m7!wlwUjYc8cINo8xdQ3}K}fP%u+9Pe=ngFcgh&r5;l7!w3RqN@(Kn zQWoOZ`kdfls%#-eN>xgtsG%V<=U#P-<=4u<5^{Pt=TckA7i5h)z)GvA&GOTM^(*9P zctz$EdhT_7@r=dtUC&D1Ye3Q~cjMeV%+clwVqf>njZPFq;YZ(b#x~mjczW4t7YF1x zh5-wsV@@X({rh_Vsf^h8nRa^!cM>taoXNxRoHwmBrvDGb?ed8aty^ulQz zHnQ7%C^b;Adu?62(>k26zdx9@1d5Cfi0)p%JRd@G^jC8~(E zxQkZWc%AVx->UD}B-`~_9>?)pJ40%-F+tsJM{{V5eNDv2*CuGgE|1}D(M>h0+j1NEcWe-OwY{OukX{P z2v&al(Kd#wG6ADe7Gvu85xmdbhV+>4Zje8#R^Pd;V+R4enJ@dq?f^&P_e+QPtyHu~ zJQu>q-0mA( zwU?ytRP=^G4Ql!7T+uA?NTxNKnG&4I=1un*hqoYRZ2Fo%TkTEVPCBM{8>gM!W80YR z4wWLN)2uN>ZD70(s#bo#O_*KBD6Ni0Sl%#QFDgqH@n z@eA}PvCDsmu$%wn+K4ooRI~OH5gXbuw_AM$#|R^Z=Pm^M#4+T@wPe-?zDMk~yuc#d-b>$+d|wvd z3VcNTI<4}#X29CTk9%O|-l=Z+w8Seatw!qzQJqOcMJvc!XP~QGhI|#WWGXYn6I}$z ztnD0;RU!I2ReedfHdPqcfwd7nNck(DNRUco;ao! zXavJ9QCmt!^DM!+I~O8Vfs%H8EsXE#LX&HCR9@AF z7WAOKEn6GD4(9GPnFg=};3a)q{DReQ-205Ew`fQEROe)gu1%4B%67c{%Dx9h2?lWx zfekY<65t^K9}qHvf8yp4M4Bs@f=LzG9?Z-D_t1hV>{w;B)2ghL528)3^sF>0@F!(? zS3JnaJU!!Hn? z&^`?cgZu>DNtI>f+qoQ-3t8O2jtKG|Ty|3X2$w|M@#p=|_lLg`Xt>>2000V^|NGVW zKeOHcmr(GZ&F)RHB-K~NJ|nnV>Lj!g2{}9JDrTQ!j)y#JTy9g&Y$-=g1_?OHVFCfb z0+Z{N@7K#b^b^P=x11SL5mGcfUS8hqe?MM$J_2+2x$){8^O1+7ue+D`_0@ly-GGEg znjIZ=jmMRQ-|GQ9TzzlM?|iv)&Xd2S@@+Gf)fb4KS*J&^Kr2VEHI}Z@r$x!&0f(k+ zMWV8&Cfd`MqFJaY8^z3T#Jw;RyB4bErh<5}vy%9LME8_fU+4zy^lc`&JIRGM&sP_Rm=n#sPBKqEp?Nlc5IrUvbcK4&%w?z zOOy+kx`)>#s5R&s!NYNGpQbE&=*$n+X<{ ztRD`zh<15;{2X~faQJ&YKKbkXJ*547K2Bzq)a~_kbdy)@$D3B`dVTL-MlRLwo^p76 zpON1$qt$oeAVJjxS)YbVg+2nuVb;e*#A_*8Av98Q3cQ5P*NL(PyP}TSbd?%u>@8IV zoX5k|E~~{1R|Ix&>W|jBtQW(29I;rrI5ky{vq&b4Hf^Z0NF)R_SxItNOyOW<-<%=- zv1ZIPw9h_EC!@f2(3(ntt+Y)FzF8)k!qhDFZl(W5dg$-2KD4U1^RFuWX|D@FKi1+q zK6Zlqx86A72HTmjQoKeHvJNgMtTx#>1&jJLH_21RJgXQ@LT1e&Cl?@0lIDX&Yx#&|XSpnoWMi2GQ;-RLw0QQJ-`mYIfJZ%VO6m<1 z8FOK#XNTRNoLbcOB>Z^r9FO`WC9vd&E87+gT2nxdvh_34W81{>*~vb?Fkg8g0IbhF zpOZ6v9yEQi8PZhPmjOSK(`YAH)&BZg%DzV{&`wJGaezpF<%F3PJ<{zv0Wl@b&@8Z2 zE|gq<{`#Z$1XhH_Wxu4p1}=uVrvZ1Sxd0ex>|}T_e92ezo@0!Ug>R|1B65(j1Z(fj`Hjh??gYI@=>}Bh8pMNoJ#-8{^4c;1w!!S9g1!jUF2hgC+98?$c;lzX^CUPiO-grfmNFBt{%YO$m^T5v<#heRu?ayh z4Er}z5P&vMfY1gifRGywp@z#6muLU@RpS%$<;uin9{owM&w93lM9cGZPoP_1Jy8v@ zZ?X8{n>2IiEX%EEMjYEw5WkTOxw4V7#=uEj6zE%IMCar3)KChZRv%Lqs7Xl;ikgLT zw@OB$)nG#>h}qA9<^TJ%Dapb}XzqC(==1Wh5m8HC_Z7= z^DRtNBd0ZJhY1J+lwPJy9#1$PK}#t^Upng}ZPP(vZ>H6-$4$bn)l}HLs@W)Qwir$q z2qdx~Qn??VQ6S3j+BVCc68Z-ALQC=N%v+V71Tr{t>k$aqmEDGG4AncDN+}@z)|mkL zNXpV2+wR%>fEIyB!D2f!vQ@SBe)ug2Xrd{o(Es$g1FrQNu`usPn1%@gaD^B~Hp)|< z2cCwH0aIE-HcS?jkO?9H^Hw%XRVZT%g|wnG8!j^z{Il0atyEwa3Wq{Ppvklp%Fz&* z-4jZl;bq_qpTXqvI?I7ZW*%w*_yH~49djtDVYKe7H7F52hT}#J^Xaz>W;DyvFMUV& zjf%?D~U(NbNfU~D!`$cinRgFVQtj(@cU75|24kU7b?TME18jy22zf2Ct zGQ}6inrgrD4#nlOhaSP!+S_R5S*c@+{n(If}% zh%bJUU*2B&uU%U^axez(hU1FZdfl`pmhB=g2==tLs~l(tw=M<=#L^kVJOsOe*+3W8 z4|0=NxF-^S51(|cDK3*Vfy|aQ%$pn{Yr)k*N%PugtelxG*xPMeCV6vbCT$lOc87vi z2n`V{-iE$yT*P`~VuGfBy>mpyd7E*=0cz<2y(Nrf7$-bYu9*nrYT^ggM52)F#!l0H z+s+J(K8U3<7Ez{mf?|3*ydGbotJv%GbOJ%>Sm*iXAG)h$Ud;s}vgr2%+1aW*`5sSQ zUh;T602Tpl77*2D5R2DAtE5>5&bO-x_1>Kjsed(y2bdo2z>z_lju)fwgitqCT@NkYeLX`L$! zA{YSLwH4E*ZA9JzNN58-JBQSq5-VgFCJJmOEtxczo>OKS3qj&}ttX?xXK(92-f~xB3fX=dn`-2qSMCVk-W3^&K${sn(a3I@(Mxo*S2t&jkg5}`QQ$*c5ys@2j$5}@5DdpLy3ukF zjqsDsTLr(31Czp#^%k0E0}h!OyTP=xjHfWL*QgEXr{;x=iy7j&g~U>kfhAMOqiqag zpq{A(sr&-)LkVZVM8b7~pfZu*0x+?JN(mUyBs_zOB5ttAyyHl*0>MojiJ6=F#~K0X z?i>=tgOs_X+gj+bL_*98l44Df5kVz9wG9?^gen)h(H@x930>DdhWky==o%q`{Usq? zcl^x1%#lDhNocciug+QP$){$n%ZJDkOBmgf+c)VyBJ#s*+{J-|RCM0HJnIQ{Eo96f z7+VDeu{G+&1bhGy1|?fi$NgxcP8sS4Os0}vQl*FFg99gc$RSM{BgWEJ=2QBvJR3b% zSZ^x43~@G@H{$7Cvb>SKWhkvqTAsg?{#}qxXG*5Zm{@Xj4RZ1o{7xX)3E@;_N7opC zD`ht*7aCxZmrDBgqt)8+S(IdSCu#P&?Lzk`q4Y17`B=l)AREjYnly&hPN`=DiJ#hE5`f{Kvd z`AbsmuK-We-Pi|%G*cr1JSr?aajZ3K)DsJZuLk}&gQGqT6c-b#XBx$?hw+~#dc!FM z$d26XH1nDyZmYKC%>L|<5~B{OQl{SXu&2Ce9@r7Gh3iOxe(}ff$`DMR`o=Q>3AN@( zo(h*Ky!|#keB%2ilkXx0yKoY`%8@3B55yV;T|gn9sV%8ZVNZ)Zx0<=hnoMkh)vIh~ zqGY;xs77v4%(s=ST%p29yftYC=1{beF=yq2ptLSvflL!=Kzcaeg1+9yx{1by2WzB06YP|O2wW-;^X zU+f)7Q)OT5__b6ep-I(Eeo>_b=r!ea-(p_ahxN|9UXBTyW}`=$#LSA<-{q=jZu~zrYt#5 zo^u@k4IMQZt>ZX;x<{76=g$1g$zAVkW^k(+7&k%8J>T{ZCEPPM==)o*LK-{*p&G4* zxu!sqdssu!YE5o#Uv^!A7PXRxH>wq}Qnc?rPeOQF3)4m3_S0bb5cl;qP;oz(jP%>A z(tZ`YSJ~f9{&U=xrZQn^b8iN}(IVJgMCP~H*&<1-aWvEO-CDAcR@R}?dwhZnk*(ol zK(;Z1=GuCap}|Y079XivbsbKk<~}LWST&=;)q}8(?qHb<$LGQrcQ!kpgSSm?UUeL0 zJU2V=({2jFyN=IifW1 zNBxRy2`fOBZL=R}b5AA9shz8Cu#*~{_D;5Zn=1}A9@7s-72y|xhX17alQ)QXG_Oyd z$kkYW8B}S&c~1lmDqeHmstiGJ24kFX*}i?oS^vu z3Mv1-6FC>}!br+-j7VKC>zSs9uVuxFeo@r*Mebyti$WDU#N1M~+u7mBe6&Qf$SMRVJLf#!MMyr-aVH^$Rpqr_Vq`ACxw8^;hym-P_)M04%`2<7~dkRMsSF_GR5k{aFO=PD*(% zq|i(sbK4LQ@M+^wv@M@?7E2HyF~YfMb2nN%r%^4_3C83vM{o@&oN}x&>KTK+xcgdA zFFcjm%~n>^gil52=85KCmCH0)OebCK1;iP}-h)nUN%q~TCXod31gIWfV0>}-x}6Pn z8F|V~XdVfzy(8)@5Z;O&*Ch^68RsdAZ!>UEmYW$>xCQwuc7jVcY5aqp0}TaJ>^@E% z4$qiJ&B9y~MaA>x;u~@$Hj>(r=JJ|FNT;-Gr&=17Fl&KTLL-sEn`gVtpsSX zK=gD=R9495t(NfEuI5Eae95$~4(EA#Vv3XYZuGC#x#2wGa4eGDJft^?j(v+t z2&%cbcKOmv=b77dlOX8Stgk+|)&z}=tfB|YQ%udqVFgQgm26aA^Eg=bmR6w;Y|^ zCwYGFg6ZZCE%L2rUF>sQUEjACI=jE8;xAs9%+e0uFFB*=@x5?&FJ{tJ-iXb^{vQ_} z54U|sN4voAPpo`xMqm*y1F1!HR)jL&sha&f@~b3ir5+}|r{%ge@S*PYxBd|K=6p;Z zV^`fv`lCJBC@sObEsZ6t%)=IC_n=I9(rEdURN7>gibW?*5a8PJu_mdaV@eDFw)x~~ zt^B7t>=m)Yn-&np5EzbtngRqoqE5r(W+V5YOEI!qu>4qR_}bjKuAq) zAOoSCQ&ig#Ei?*NI2G=$EcbO)3J5Zun)9T3F~pRf?Ab(@-GN6fslctk8=3Kp~{Y9FG@*0!~JxJLE zcXX#4cO6L8sCjU_&+m^Vj$3KDl9>`MD*Ie`z}trI{fA*zwKWo|&U~=S z6@910;>FZ${Jvx%Y~qGEx%xrO5gC(nTiU#N%3zdyQAkeKB`>GOPqwo~`t}t)H^FwS zZ|zTglUX!$Ovj2S3D&ocnP?yCsfMc8I=TkDsm0r1EOWBm2+-r71rDu%Hg{EI$oXt3BD za~O>y4x#~khC0cM;b%(@;`@g#NrjlDbZs-l7 z=$rJN(b<3Ws`0v{&-V>z@VHju!ql8M-r?tzW^H;ZCbg-sLk5~4kXgYZAz~2e@e2xZ zbp>P9ct7lA)->$oFmdL->VqY$Ny1D)dk#%nwXMFInVAo7N=F{vmZ@T6mmTsGqY2m0 zcHe(6K2~`&UNPlD4;$&Lpw_sz=2wZLZTa52dJ~-WRhd(0ol*(=F%H?m)LdS6uwX4W z0*eEk`T*lWZuA}-c#kP zfyS1hSrbh=<+dzI=U_Z+Pqh?{6lb(uVGR3JTnE_v5Zrf|5s0iyomL=FKW+U}eFF-MDqW`}KJidc8`&T(1rNfeAXX zTHxbOvZ11Lr>J&#Icb%IX9dd#&d4yFY($XY-Hg7D}W-Cau0~tNgxF)HzNjKbaaEFfF zYT}qlQ8D}L7$MOGwv4F9l}}4dq~{4x@G?CNTulmjSUZUn%ucWg@?M6aUAWK&H;#P( z=5&57EQfH~w9v_<1A5$9wD!=E#oZ23^R(Y7oG&QTFZND(|9&Ma4hYbu-iKEHq-Fnw z;T6K?dIpJUMjMve=?dm7qQD|;a04ah3V?G~iX6kPHRK`I$MR5-INZS5K}+3Ye06=y zpI{Qe`!qS?ywsI{Zbw>H{NM_r_!%Ao?E5#Q#bO2suIDJ6iA2l1V_rPM6(}6Yi^hL< z0$UIJFFWVe4FhrY=;nt!oj6As-(aF*AZxBUIn*7=8SUsW9j!jDU19$*9M1V|8k9NU z55VNZl`JWLmqJEQHYX{?m7Q|A*rhmcam@#Cp73#f;sGagA#`dh|Ic%(k>D!uwn_b? zqe>WrZeE-midk6Iuc0l+v7Z=nelv8g9UdbvxVS*#=+$c(F4iK*^0dEldhc13rE@vLBk#ooe zzbzQObfaJY%@-S+{0P4tjjI&iwS0N_w>j3Ue9PpEZ4z-JfzT>yIVr+N6hxn>ie-St z_WO_qF)BiKWS~Qt;wWpYjcvGSOqbI|+-77hyDq3~6V9ie8`|v;Ctf$)TPkkYYl8mW z05uEtJR{J(Ea+XJqJ(S8Ir2#V*IYsvr3HxC7+aE}ep% zv9UWRn_l15900Dro4<+&v+5Dshj8H zhi$_uS>@FSV0Ob&HKHpnBwuvQE;xNR16tcrUcOO;Bs;@{%G`GzM=CD%p2R0ZxPACur|o{!)gg6$B%Ean#<&0;Qt=c6;2+1`}~I> zoi_d7Bf9^?05iV!^|mot_1x`vg)`?kN*H=I&Tpzhzm}@HK$LVf(Tq>EAw?pcn=BNy zX3fg&z=3n;5T5O6I-oMa@dm9k;ckI0lg&nP2t=bqYIi$ev$HO;$ZoAbEQT+zgF#`) z633_5`rRrJ!y5kf>$}bIHkYU^-jFT=rE-Fw<9*$F{My-dj|;kujqU!a>+@+ayYk0; ze?6-TjQsh_TuRrMXNlQ#o~b(L;qmf*m946u56lN{#yWorlh16%S~U0BhtE_)*u-UG z!)LA5M3>K8#hH%JRy9PI&tB!3j?Y+CM5oVC1$xp{%4)_oHgnPblE_?BxSqpMg)Et$ zY-T)ZBa^+1>BnfJ)nw*8TRm$T+if|MJpiK>O4;PN@mE~NSLMQ6Nr`j$wbfro(Jnx~eVXBLvdk@Z3bntExi2WAVKBiA_1*v7|}z{JNCyRO&Q&kz2-hAeD}+ zL^{LfJEP+&`{uQWt<2N#yj?&DnV^9r!Qh~fWLa9VY&m*J%GtSUDfW4!p5*x?)$H$8 zbBl>iQjJ{`uKJ4CwN_4wyu_k4yLGOYh9&D^y|%6B)CZKo>N=F|2%_Z1@=QJ6Ypa%_ ze$lemu@Y--jrzrMd*!k670p(q+fc7Nj_O)kZPvAzJp}HUwL9F9EF(gKc-8lZ9x1dQ3!AfqQ+?E~E}^68oRf)UN@&BHlAVfMwss>KQq81Q z=!de>YB0u8iftA2+PNHygFJ3zHmvLoD~wSq7t$GGE_zAQJ!VyYuO(j0QFAR>*E_Si z-0-{#o_jxCLt=+^n&)29Xe}+C?(EeoLpR$!?dwrjL*IsN+c0ysy2k`qhMzcCBej_J z)7p2X-HQ3Rj&G&`a+sg$U?6AE{FKx?e*p~t*C=i)EdvQ-T z5l^5AIPKp^%!qB80I=@G0Fu;t>siE(l0`2FUsdXsz5Y0J+OX@xcMVTqMK$;U#F8_- zK>`MD12AY1^4K6GxI?;hVFH@q*J&6MGhkz`m{56U&JyK?Z#j$QH{Ezj`YC;{= z==3UEyy!-Aqh<{RUks1u{at&H&hkFUgl2Q!pE#v=>X|Aa?}=GV3{Q&#>=7efuf9e^W+$&D2vtS;IRsZotBJlRl~UA5=4K z>Kr*ZmgCmJa&IZBZ&_)x)?W6=ohE)SRl00m&5C`XX*C0zkR&PFYCW@|&am_Zw6_y= z!cJO}kcu<==SX*Uh#*QUByL-drcUY$Ot}|+H*}|?Pij1oy(O>W`wz~Mzjx<9oAn&} z!3sap^eJAk#q|)7W_Jqwta1}ifDds{*}8IXbtM4^QQoMP{JIr2x6lFj$xU2eOWZJ| z6{XLS4!@nq`A6{();hqTvR6x=0S__w!Ri2RW;-u!4*6jyQO_ldb&I+to$hL$G4KON8Q3I_QTp2<_8(AN#p zO?H+}jw{kOl-nNTL}ctLEt31O7_kXD=+)G-9Gxlz$lvOn%TZmU-gWG~ZQeOQeCKLb z)Lysy;H_ic=uEh)!NU_JJBl!2Lha~c3M(t`Xpi0<^o0VNFm`lXtUeULaNmE;N_DGJ zzr9Vxo?eK^#K-j5IMW!~c%06=)U(O6%NG8(Gly{-U7=kNb7m9nj>uO7kwa;q1{}uc z4|Pzv4GEjtxGTI!Q?shAoAivEWtD1bS-W!7>QEBG(}|gV6A4HUVqLzr8X!%PZHOrX zIux&VF73AY&2D|srDDm(uR|m^G})hPO+3VZ)yTc_JNX28bD5ivOtCXyl$E20ot!IK zF_0F<{Lf%|vF#t9MRGbj4?E9D&qjPAHj;7jkMO-52ZjO8K-mgP0P={n@A_0Owy*}& z^rrH8y3WCFTEwGEzs|aJL^kd#FU%8Tw5+5(hu2k!CoxKI6H2%@v;C~I?CV#d6#z&A z9f)NO00SRr(E?a47qL-lT2=}0gt!2>h!7=wrF)YQ_?)b^rjYnKeVWW3MQvO6CxaH- zIqim66Zqbd0{=W&)7{#N98$4x_7eNelnVmsM3-ySf=Ec_;cjS(1%1S*da>lvPhNQd zfe_L}devJSkm7{JL5|{S-O9Yh@Uz^60-1Vsmg~8@N6RJJkF0^(>5P#cNMfN>J_$el zrNV}&ybX*-TmT9(#UvY7_YrYFwBw8kLt+F_cXyzy>zm>jGYPRKZImrg_1kUmf~)2_ zh-KmnC~@o15_Q@UZ`FGBc{W=5@D{nYpNkH>a#I?THm3?m6WbsJg*a}Neq#KYq?f}_ zJ))9G7*s_7bM5=(ngmkN1OLAv3E@E=d%5J(`y93;jHBx04L)KdiRV_MWRe|$yA3jq#M7wwks*3+ zlJpy-_kt$kKZvIBhk9U<<#Zwx@CN0-?8xtCIz*gGuqJpJ<3$o9oa;sKBuwG{9e^@g zHCTDmePjxyPPppNq{nmY%oopLGe8oWS8e#WxE9f*;Jgn0r-ZSj9yW=29&(v6T=o_H zJqks6{Ha?d7cq)fM}OSKo60v*D0Tm#^fO+OVp!BOq|G;`CU&io-b-oc4(y{DIXR2E z3k8!g+*6(}L8TdV3=}NOa;eh85K>|R_=R=Q4MMSA1Y23MjX}yX3TmsEMB}K&PUCi_ zNXG?Qz;sQBa?fnatJFW0$fEuOf@o6T4Y1#04cr zlx#UP0-|aNt7C-{5Q+5#wxe?1!4!6$C5?e*Y z`TXxmgLW7K6fb(u12x?Ggx03DJw$+kSxo>|XJ%QXX7&F?=$(4bZ2gz2&zf874&6;Z zb6TX0U#4d8zkcxHpB1?e1;ij>|6GeM*;`O37lHLA6^dN;w6HFyWW&)(j;*zSplj7g z@pc!!;fY;<12p&lMOkPZvI#vZkI!2;;y=QZY?)#eFPk^2S+cG&83aS&y@32zIOc0M zhN*KEBnpvm;d7uf<@pxbf>>-#NtP#b{ejq#A#qeOc#BBLSSQEH&KRJ`D-xOsc z+hjFz2VYJ{HMLx)BC#lhlDm$4TI9U>3m)}?qerp z8O&h?T9>57{D z0hUNh6!4_n&98<9wRCb1`ZXo#0aWr%W`{Z$5Vk(;dK_^~ zQBe16#nSaVS@*R=vSHsQNjR>_9-i+Jlm})qi6z_ycSr!ai%v=PP2w2Hj@kr z;1{->f)^U%ksEqO$YK6>kszbWs}f8;PKfZUi-#17UP8c0SqmcA_Mcl;*)A4~O{C`W z|FWg}_a+G_6$+*4;{pgsDHf`V8ukE*1EMOLHkIiu(hVb$Qn;v|W4`l8zN!DWOrOn5K@>6>?J22w49kBRhSp_2Jc^0&xW z0r(cE0frzMaxGK&K&%66g-)KN?6SYTAk-MqTB|~p3B#QEn-rZPi2+8r;@>0T4sy$c z8=ggmpu zHk1JT{w-&RS_zZ^v;od!yaO^B9Y9Q=45BAs2D||Ag)k+ol=oxdKoJ7CNrlLMPxlTb1x=fXe6-!{UaWM_yN|nc`DzT}2 z)fvFS>$dyndkhpNW4TA4-6QtN(O> zG$}JOFBgs9vn4hsx+K^TdT#$|QuEY){~&9Ek+{?c@!}JD;*t3cG<~2MS*T9G{Zw2@ zuoB}>%1fjWx;28d+qoj#^#^PMACY>FRBofIit_@Uu!h|KSvN7~93WnXb1VMi&5PZd z`uxbBt6%U1Kkqjv7J9V=K;XiA!=h9LIR6#cUfNFyh#B#BsP{?$h+`rj+pEn11UMA{%Fw zjPD|gX44PwrIuetMG*4jPOt!liaMxXcJfbEtM>Hx+>CK=gwJPxfIioPLa-Nr^dL4D zPU*hp_2dll2sRwXhS?l1#wXk%a4dNe$E!DkR3IjzMkIz5qWA?#ifTY2FEWt0>LlFW zl@b^QPlTK#Mtx%PIXiwOw8WVDtf9wta%s}FUl}a_N`~a)wW(ghxuyJz2MQ64@--Fm zY<(8!A73{B9zEP5>aR3};L}8}ck?;v=MD%YihWH<1kg_b?!j|6d)TE7kPp7|?(+(G zx|MLE{{p8nS+A5!R1UP7Ua$ltQ=S0H+}cp&8I1$GOIiM^$u9oNkM~C694wDA7dBm3 ze|^&f9?m?u?=xq$g3XCt<36N#H2pG_w_=mwUJi+-S1=`UD#fZSF~vFu-l)hU9x9`? zpLHr|P?p83z%jXx8kp(2Nqig%$qbx=Yo@YK(ZC3=-|QMXh4%$7e+SQzJ2Z}$g07Lm zh>pIdJSO(l|2>00?oTBLOunL(Yvr@sgMz7&XKJw{_l`Wug?_1kn;xMNTv^!3bFOMK zzK;1S+Ok=rx4f4RfvtqdJCOy{;BeK|!)8wxfe%v5d?G6US zITd`F-r0GxO+PwsY6^UNy>eof3F!rMt=-{w2%WHQb69gci%q1`f{cD{AYa)MI&d!9 zC!fL}>DX?yxFGg-iZd})tPi*|Fp8{uxxKMD=`8arCS`w_y!+Q4!M$9hQH~LRO?$d= zZTc$MCn-;kOt$0X3q>b8!^V@gTtGFDO3vC_Q0$c_7~HGim`&%HPqfF@(k{Ul*rZ5Y zEH{$Jzle#`z_&oXpBS?KERZX_(oCiaK_Ws{M)Q3Doqn?m$Xtv`S!sC+fp`=+Eoy;W zwviXnVL!Qy&>p$~D+9H8?_PvDv_siD=P(%6i9N2p!e)Nyv>1Dy#Xhu-9G{lYjI60DKV{K zxq#v-%d$=1d=Hs0zC&2032I6+-_Ze5f#TQ%_I=};L-J3qVO3_d_P2i7EzC#8dK>#O z-+xz1LbiDKmo*guUY~f3$Ub^O%3plpFJGp=42W^$Ut#O`QrMV)db4?AvxK=ZIIDwc zxDZSd2YtWqd|!RT+g!&PT!Cs4-*qEnmk|>%owDC`<=L4Tb2RmK%^8Frbl``t&%54E z@4GDI%zlEV>JpHI+gdQA=o_Xgc`Wv%vt6?Lvh7p!MIbQ3cv+AplA zm6sFZ5KAEGGIn`4sG3h}Ilkh((wp&)Lrc|+ev*!cq_f~#zQ%(jMdcu|({7u}ngqr_ zBNNKX%kuw=v3HCSC2SUT$7^icwr$(CjWxEdHMVWrwr$(?%$e`p-0Yj3+Rrs zr~g!?-s+@g-ntElc1K? zrzVtKd0~4~{M{;p9U){Bz0ru00k#*Y@mZxTq)Oy@RR%P4 z!>Pl#XuGr}*wrNKqkNK<;So1pHRfj!t+5iBP5BVJ>Nt;Otoadi&;$EQ>~i@~6vduP z_w!2x{Jw5(}iVEv3v6 zgXb%|IEUdYI&09E@mw6}$a_iK;BDaXaWrmOPhXQtlt|?Z@rhV4^gY2Ib0RmiT0-W! zZ`*{**XB-9dBA=Y9VAsM^bs^sKTIm02y#oDe|VEofIwofA~7fewlO$(<2D)|sX@gs zERfThnrEEal3eWHAE&%mZI(FULCHLQIrk1+5gQT2k*iN+b*DI!_y`erq{sRPw8smm znj-xY>FF85%?RRkN4)m}>3ET)K`)ByH!x6khR(L+b(J)#&`UL3;n!oKhkJ-&bRMrH z3M<8MKdj%vz43XD`QKBN{$)$c1rhr0vX=slV1eR7PPGf=d3+}fZe)W`bs*&> zpOQm~2~5PzE$o5)nF`J)VuH<&=|&SNv zTtLLeP2Vn+AXBsp>l@QH$v!6=$nrzc7z>kqptfxp==8K&IyKokNevr*eUR>Pt_%N& z3CvCPP|0K)%b;&jVjq&n8yNWb#DSgJk`vV~hx$4OBs;SbJGWhQ6nUP>BO4XUIjt0a zv#)>V_k|kPqZ53S_(1{bEj$dFy#X9s>1wHJed%A~+CC*(v>s#sFUh&K4_O$A3yPx= zyxani(~Gh)LKZV#>t$W$Z1m}@e-XoAd%RtC;YvHzcqD`Jd*id&NDfl@4V3Nh)NKbe|2V0>zT%noa#ADMwSX3 z97m_-vvMbDr`{p}wZP91&o)N=h|(7!G7*j$_Gmhq(jGF9zqlO%qCL5HprTNpsJjr$ z3Z@_=DodtB=`dkB`ELg#5jh>B1eRW6U^a;24DVAm{f+5aqP_%+Fhn}9h6ne3#E&!= zh zw)UsTlcYN0yh@~&n$$~TW4NQFeil{)@(7|-5wTOs-181_GEZOedRA*AY2h^MefK<0 zi->DtweT=xp#s$Q1o zOS?en8DDr8>Qi~epTLBymX_vKG97>%D)nzq^K-W72FF$u1T6M!SwJzJtrw*2v|*)fiw zEwBS>s#lJ@6rpWnKUWbu<;C94#K7$1ug(=;my1Td0ZC}=jL=%TL?@xD=O~7S4&OV#Qjunk8B3xg+*@b74Nh@MF z*L0MtuwP=AAg-jap<$}z+YWz!uLQNy&Re1J3UV>ylz6o;CxKnJ}lq<%?&C{KN@FL`l1%fBTZk7h#Pm z(zpxn%=555*Xw34ekk(R!gfvlO{-&7(11dD^7l`Ti3|K@Vt|VH1A-X?>qp*#kG{at zNz~zE9+9}0;u>Iz(Q%`b;Co~H+CTqrmhuDZWrx*oM0FqrC(^nTvL`V3HKsvFJh}PG zuDw_y>8Z7oxBvu$8RP7vrguk+>*`oxz$RJA?XIlVg}y^zcHP5gxkcZs68jnjF!it3 zs#X3`VhT5-vDAA9Ah2klDHI5B5Xvh^zopOS%F(~t#aeP6;$j@Mw@68t1J~mZ)x%^( zV!;Bxf!bU-16N&A{4O=9&tMhJ2yY7Ji;4uRiGMjqN0fmSPk74R@?XWR1cyV#(6nb@%Guu+Q zawlh$FEp4itHZjhS<`&va$aJf4CkDaRly70Ap3b5PZ{ak_!!yfCJ?u_se7 z=7l3H{i^H{EcVlKHXVbVy!o7$D~4sq-6i*h8tY+Wvlk*>Ss@t?fAw=9;3|Ct`6IO>T!e2G$8)pJr> z9{b6|u>VV+obtR~lmAd8%$z@&#mVsv5x>KKFDCwtgt(mpuw1qbRpGPhEUqe`22yDPu%xZq>^&(5d`Y9RZ;wN$?%nbtqV2 zQO;ceeVGyO2ngm(`wHCaZJ|0qb^^}@6Ot&k?k$sUB#fhk=Z2Bd`(E0Vg!-4t?>2H& zSkf~}S8<5--N$AO&_&kjd{dzFnHty^1ooWDgjdtsGzB8XQ{B5wRd(GNM9oQy_y)`^ zL489@nRP*kie#hG5Fg+qsAl>wxG9^-&i7^FziHhsec}Z&D{ETkae&5z6`GHyj_Q<2AE;5Zgf>Y>CrP7w|6X$IbQXGA`NgK zA{D|KV&7wc(cOI0c#e()o1TIqad?s?!5(139sAui<@r3F0ODrcW8-&u`wgI>W%FuN0g>nf4l)6TbNIFdZe)7jteG|63GUlGh1G?_ z%WfY1)0KiNi!(L45o2oXJ9y16P8XdPDtr@DN6)3sK8=e}q#v3Z@`$SAgi}@VZ}Bhd zXbXchi@snjD8Ie=XLY#=ba`+Ai*q+FW@QGWP5L8W;9-Nf&o%-4h9}RZ3!zENhjv4m znhWFJia~!7Tx3@ux?#Zu7>5~V%Ek=W6KY`ip$;M3lgQ;4`&YYXpZ;oPZ9Y;TM14rU zD>r~p>XYmncoq#VB74RQBw!%mrpUNlYp?(6mNfs@DBU0&)L8MLXFlAO>v38h3|BCZ=q8rqsk<1ivox*$8Q0|tR`moC&&}&h0M_;k zqw{VzTGH03#qz|Wkq!jxlWQEn-}!^hve#!=GTbt@;t|}PqraP*4Y7Jdl72^Ndm16F z0)gy9Q~=YHG#6ceKBna^JNSaB_>B(bKNb0Cy#F?|J)}Cxy#)|uMI|zCKVmBHcOgR{Xx_%OALo0o+upk2M#;6 z6i!JK$G>r2GH=SSGs7_B=|9dj?)!}29P=N zp>0e4#5U8G$YwFeKz;h6J5}iPuEo%4a@6j3Ow=%rY7axses<-O_ibp>aM-zpXKr#9 z2sbmXo+l7E!fVsu{h9Y}gzUmE|6Z7>I@OW#u<+1E6Ukna>S}6gWWIEHk~QUarL`Fi zxJ}2-4ms+>6cu2Uj>7U$o)vx>81QhMCE^Fpj?J!BKt=NDyZ8}+ARiW%e-sdrbvFy> zKi$lk{OaE4|wVMZ@Qmf=nw$Ve@?&0e^WU-88}+$*&F;dFt#wW`JZ&q z7-f&#?!Wh^{w|W=Wp7|?Y;9t1U~NP9|ND&p3m>!#2%WR+?qY)GZ20A_X#R2@@&2cGH?ns8MK5Kk{J)YTc5=9SV2$vG zyj5=W)Rf}Xx_%04GZ6$3Ad)O=LgcbIWt^H(-VmN z+IZWh?4Ip|?Po#LOl@$%s;Y&$-OC zRigD8N3Yi`EsNLSQIwb27Q~5}SQYM?8@dg;s|r&(e=N}w;3rpV)&JaXcy+0Z1>HH+ zy?s5~0^^-jEP1tQY*y~MfUdMmv_wwryGYpG=y!jD_);g^5%F816!n!za(xTOcd&Y# z=|`^;$(M%wE2BREohhYlt6D=fsNgL~R)$qBm9ouD4LLmuB6J(0Y4|=aEOn#L4U*I< z{bU~sE_H{EI&{_p3syC;Do-UhqNO*}CddbmmZON}qlddMmfpC_Oo%&b(uOKj0(1g1s)!fusm zpP()?aI(&9<)zOWII9=pZ5B9Bs$Lwv=bNAw>4kUgmR<1JvM0nRylk@nmVX8^n$oen z<@>2=AtJaoP=`ocv7R2JQ4vj7JfK@p)f7Z$Fpk@xECF3|OJWZl*F04RU2M;nE&L<+ zMG#O6AF+m4-5+{#8uoS{gozOyB!)~a5_h}F&K}F=ut)rsfk*M`ehcZUvavGuu;2Kx zfE#V(E{K8DzL>y!QNRWnSTk(f_B1!b>-2cK-(6qnFmYmee{L(xu6>T6whCz@BHev%I;o zex-JXlFlNr`JUwHftNiQWK1$0Ae|bNDD&KRw%A&AAT7U+);cZIIoDJeJmgj8G&VSE zEWLZlm>nhRU7E=Bpp!X|{|*Pc@@Qq2);KWX#*FE#Wh0Au^$j5ToY>k^MZ9nrC!b<5 z4(E*tA9HzCLZyHfq9?dDI22lF@{ziRd9y^8)yt%Z7QqCPEIziSWRix z)X$#UPJyQ0I@X@1yd0QF`)$U%kj(b4lal)PkW0t+p`TsINd^-U)1jF^npf!jHqvM%Q?AYC;+!}be zvSrbku|c(V1zv2tWvu;m;VP+Vq0jOS;xm0*a6Q%Ug=t#rqq+8fH}sL#K-O#3>f(S z@RLorfLw3Y$zv?dW&PS-#Q8)i+XW|M`9uVk1Zoa<4ZBpUlc=K{MiPdr;RKGZm(lCL z91i0$mMl;CN;GGT1QPh>e*z13Vdx+e%l3SgW*NSP?CEhT*B76kt*THk3|&39wU?t= zC>m5Vl0N`xbh8;hSTAU!C*M+f(e!GeMaH2HK6(W4%V}|Q5 zD_#Pm%j+xW_bq09L(n@O*HiS~h<+m56aBI8I2sU;wUpJd+|AQm`GUvZ#SQ`-w4Cmz}d8t1n z?Dq0m&N5R+D0>E%{48eWKob9GK^la6vB=sVDIE72EzxBV`e!pjnFbqr+^aL(R^72PkgHJ(0wp;v1+G+eAs8f5lHwISeCA3H!Y+stV&j4k`vDQPS2P4pt&aV@br z?UZR3TXgoGfAU$pX=@yj5*LOJ%?PTEtxy|13`=h)KLC2TNgYN_DpR%_qVU-y6yxjY3EMtD_ebg$Uapte_>H1(WNP#TgZ`;DjxnuK&t8f(0^J9)!1 z(fu+(us*)I;5)wzehBOHe`=3U8uHY?4S7X)x5u~B<`G)`+}50Get9ufo?eqD2;Q>B zx$rSVUSZsbuHhrCdC!1}A|cyp>{bx)yVLj*9Zq}v5u&1OPg;hG*OGr=@LqAq%*HO|T zU_)}kxyOhN!H4YJK3PEWoB)I%gndLV5bi2m>1q&TUUY!XMa+~;9lxUH`GWR-q5%j`gP}+ z`YWXX%ubZ0eU!LV<8nk9(p1+;Q>v_SA1k*-2{>a}HU= zEgz1;7jQH|?KS$6_Mpb6)O+RUD&imFbP4mfh;oK5$h=ez$&Dc=R1R^|{_t@wz zXOTe_02Rkv+bna>K7{ei-AKQKeSrzzOIO{Di=awWEGg&~Ga3}TWtb%;r8R+g9?*a4 zUrL}Su?rCphBecngLKxoCs?LMtF2?~BUFfWzx56L{ypq;qPbo^9p4z&Va{%~JaVt% z?D4`GnzcO0#P}M;(M`rW)O!Yn$Hz1G{W2um#Yy}jX&ugT61LX%R+ZZyO3)&VUfiZh z>~wjraD;FT@sH?mUB9X;*aBr09<5I(AgrB{y))s!n*>w7ON^%Py_)WQyO!;ek>oyi zk@We(lIu5IX{ zWp5opf6E`Z|HGE&muvr@Esxs&qpqpebCBm9-NrsLp<87HPfcn1&z9%c>xzI5LZZNm zAgL@t8Kd^tb0aR%OuV7P=3Ps(z2P{66Z?pf>FWp^F6~~|?(Yg$GvixN^;-aKj{mKA zwY1SsjM{s@KR0ueT2QWiM=DOzkSY>Ng#>(2RT$t|kmhM%ew>~i!OE@`Y(Oq2G=ZGN zjFwE_RU=C4)swcMg=&SRGcZQ^6g`af6@b&y5Rh$=7Y9xr~aV`N2cqBu`^O?H0r!~C<#Yh^T^5g9pUilq35HPOm64}j(<#+a4e%@<^mPGr5|#IpRc8_v>G96z`F3Qh%)#2Ttf#(cUQ>SC-5gn#t$!Gh_V6tvm#|Ap1@zJ0xBG8^b5dIV;J!i>OBlO z`Wi!(Ih4R8FVpf~r(9#olXcWzp(}D1c($WZv=)ar2?l&Wlr1=h1!+A5t5TdprAb?HbCyv$_fWllp>FK)YsZ~6o)r>e zb-2Q6WO7q!a}aqzFAW&*5sRFXD*V)sXi?tJNG=g~xQyHUUpgLo>&*E)q|97L#x*D-(-c(pZia;QjHt)cPXc$= zrPAV--5u9pN|VavesTghM6DJ9c`j1e)kAk?Hy}JRyFW8>I6G*HE2lNG_&lN$(=J0( ze#yzeX+-}iGnkzXW_(UN6y8M(DqwQ-ar)UjM( zG&9SEC=H9*Vc*?7b<@qmWRoE}wU5ssZ?;g)(?VUvBtqq!mbS1*TQEUjw*P{&nr;PD zLBuW7|xf3$dqeO2W7E6N5m_L7MU2OOIZt2N>SwSSvKNDTtf8;xL|JwYcX1l zF;zWcFx~OCq!#VNG6p1n2eWB-CSpFAxRYwN9NvE=y_;VBL_~g33YYB5a)Y$6fR&{@ zo1)dqbMRGHU5Qr@*f*ajDzeNUZv4=?>_Iqs>$Ck)n2*g9Ic25lj{{52DO8!HUB>iD zf(H=kJN(IybzF0E(Dj8gx>@72dlEgj;`)n>YiI}POIAo-2#(kys-Piljueo9xwMX)v(ZLyd?bhi>@lkgDKT%&=;L!^;2EM-MF)nWp27IG;BZ^WW;uoF5FMmEU|l4-Ei-|3B89|2tnBRjEojVu`?KuPqJ4lT5f=635^m;sfIK z7UQR4Dp^*_&|s}J)C{TZ10m{LVp#71+ypY3p1XDuOfxh$$Ak8e87{zILza> z6I9ZP7Oo=S1i4WSMkbI_b8pziHJ#eM#cU$^Q7~V zMVBP?_Y1{t-|X!fVOEx6bKTWaT|LS6qiqCL*awWFCiuuSgK|hdB%mN9W}{>a>le0?I zB}SPntV+_X>FOx@-N`iH;$8Ul8<1Ob?`sEZa7GMc4<0I`EyI|zF78bjBx|ZjzQi%( zMu`2BEm*~dl~HaDWS1;C$;sj4RuqGSy>jznpBo-@$v-%}?K)?d-apj;yQjK~?peGj zqYgqT2y2{^Bd>~bxdP$oAEfXF8MlKLCwxq}>K<3xwEsU~z2(nU#RXDJx(!XK!vll-I;bml z?M~CCCPmCK{Xu_AV9Mrz#607cjcc) zZuZ4_N)>_qTyCA1H()zgF1T|q=+`A{*8}@OcW*R2cfI>L(*-I5qkqjLQi2!iv#l;q zlb>$544|zn$@P$IubAn+KPe^yI0D8Z%VEdVLouz{@6_t)6s$oKC)P}*kRh_T?orEP z1*l10xd=(s|2Pyi%%bjBsuaoDY1vz*iDnvjWfvdAceBi?qbr-fqQ7Zf=ULHj-2g<> zLpup$U>l)Y)T$|L zUwc4=0k5{rChmu#OxWyjkW6Lk@0Cx)I!!F$w0J=KKGRHS>X?Ko2wQHg=)eUv+c>F^ z6}0n_en&TAn+)bC1{fk8oP-}8){Pq;bfG`hig+t2Ts0lLG8#DmjpzIIY_$vKB~Cqkk@_t$nQ?Hoc_aXLySg;!LkpuI zIsU>o-q~^s@?2Ii4cv1Me%WZF>prDWBhQXn`O&EK%h!p7V)O;%F%C5-vL!KGx8%^P zTDt*uNrw#DgHPOJRq2Ns=8=MTVlXR12_6oSarR@($5~@^7qbR}?GTwquu1A>4*(R7 z;FY7Iv~wrt&WExpsY)5jtj_MDUr?xs6-u^o(l`e|8a19IH>%CgyR4~rer@&2N!Tnp z(xZ@2VQM}&w=4V_OcTMiKH-R1yTMtUT||%pQ-uq{D9EGqXdiz3f^#kX0)%oB08O7zMAti&*-haX%K=io9jTApGSJxPmPyiGh8mc6O z3=2FwU)S0Ga%*cn*>$br^Inc)KHXn;mpgZ3V-!Z0HVV9|XQmsq_}O)8Xmme;-n$a< z5O2QE#Wu|%DxsG$$#t>pb*6YCH05-J5OT7BBXSnh_nO3&Q%DMRrcnGOv3s$YX$?JL z<+72PU66F@en_O2=8PY5y|?O()+2Zty3CbTpt<5!&9Vm0ckq$*1{Sc3H{;Yt?$oQi zHqDJuf0jpnKRH%p>cP2__?})mLDt zBzA^e(a0c;3Us@h7;C3=u`E4Yqm_OTRknf^RMm&6xhaTOXqw5e0#@)O#-SDni_>)4c*+OwGlKritOmM9v{LYrI%x4~pb4kbI#rHP)q7P-W zTpKK-ocfR@p58nuOh4yH`heu}*^t@j{@#Y7kS={}t;p7CnP0hDC==ZylI_%UZAwPw zFf35n(pv}@A?80&w??G3awHkg;%vWHSUh<-}toK-+3l$Mmi+VK?SGH@6>l=PaUl{0;eO$O3P)fJ-DMg7T z0(gN7?r#QZ_-FxU5D+|W_(U1YCNSuh+;3|^9{BRUx+lrWMj_^*BqgR6`iZ;dCI8+6 zjx8Ue`1;jpI?NF`?4ETZpOcmnf#l2sheZqDPRlqv)ZPhqw;P-eOhJrFfej}I+t?JO zR}V%IsUo78Vgt;nv_-)8wHLL|pfX~;M1Tjh70+E24k+I>_HP}a|HVo-I6 zae`wW%uI$rVa*ZWBo#p3YWKzF_~Y6Q&V3oBC1bi|3cvNos9u#!NqeVosOJjCqxw>% zzlaG^&Dz;rnGGqfj;x>~FC%=?H^bT>YhMPlHJHl1KEtJ{mN$xWa7G$cVV1RvK>A6G zQ}eHSGmHpxoK$WPb~V<@N#^U6p+mo&*4j^?1}>oxvx+6P75a~QcoPJ>0wTmd>isZf z`b%L{r9t4(TJnD+r8BKieg39jhZQ5TzI>?tHAWGI?BY(ok%VO|4f)CWo*eyG1b0;G zTIlIYW}BM? z+{{8ix6Hb>U+{4wsa(mxq;Lz`G~m}I&22{TtCota0iym^&>zJ9a|yj*wsmdO9& z*9;ny7-yR7K~}twaMNmXpCl0JrY*;r3E(_VQ|D**V##Q0Ri&u$1-7Ud`vj-36ZnLg z$=D^#oH8X7BL;j=U&&=9bWI?_!}tl`0jp%-p#V9wpXEPI>zs^FQBgf-}&V8(^qG>=cD zW8gk*viv|3Dvu#Cqr-QYZxInTLm+;N z=VA@u!VIr&q_9FM2mm^cq`3}7N#O~;OR>s3#n_Apy(tXl%%aDpjvzDzhXrp^yD!t3 zrBJ~i{-f=+r!i6I#Wxbg7L7&eebk;;bt5OmHrdUu>5dT)iLie|NZ;6z2XJt1E&nG8 zsumC@?-6(nKZ*nu4G4xy2DjE9aN3dxsMH8ZXnpbl{`JiBrm1OFu)#hXxHfH9Z9eB(CY8tYI+?F%S$x@%B^i5PfUh3c>ETt)Ol)z{X46>s)mBp250% z%MR$Rey6v=jV*)WW+PWubnt0^k;KJEbF^H(E+qIf8Zrr|NH)V3eR`b{YE4x8=$|M| zPmOspcp7R8D(t0bn2qNgi%-tpM@uVwG^@yNL}1)f%^#r02v-#q<-SC<43|F$2~Yp< zK`-(_@5>}4O=v&T}F7k*rY1A>Z(ATG;kgAO2Eg?|U{Sn4MwJT2r|0m_DBPc@>M z(v`F1U6%C3BrS1e!izg&Q0CE?8hQI*uFvdVL~>_bUeD-&VYZf50NX)BaBCS2gYd;r z*sGBhojm>lckp(v-5I>t2^^tSTVo#)-Q|Kk$9*L;CeSKDXQvk34~NOt$Tb)>B9jfC zK8BVUoPkAg1c0KTyuz%ft790l?yQaZrN5-LxUZ z%`Xn4=Cj5V@w>L41=8jm?Hw35VQ7#CKxebwV#s!J3NLy`BF2UuMnZTjd{#3u-Y8=YZHOI&Bb2m(mbFYbi{BHyqQ>Qwdy8dZ$Yvlt{ zuv*-Q-(cp^tBkvFYos{AeQCZs2FM%&ECzRyYbb4O5d|?Ng00^{O_jUl(M5YtTZD;1 zU4rqB-4N$Cv)+?Ha`IsRxP21*t=4FRWPIG7f_2xY?PT`FQ{&e5C+@ zFIZWc6FTB)%s8w#V$zdmlY*%vTW%YRvr1L&+1x(xu%%4ysJ zc|rOS$&8Ho+3`RYp48a#&)KcMyTgcX?Y2uYPta@IsahugSg+!a)-a=X9@p^V2t7EH zrOfn=z%1YbhUqYSqYat2kIrKUd9rLen4e-Upg&;Ha(neYURHFD(z68Bl97hrH8aI> zM=W*7ctu~Wtia^7WG_=_hW$1VX5WT2R|Q#3ey6FMcjy`s`x-ZZ`W4E9~8KHUhpLT$|l z${ZWwf}8@Z%hBzEl-@|m{)mGg6dO1mNTYgv?T|aX_=8hk5044TePSd*i&Yi3$4?*i z)oz`3E$Q(zZx|OgWe!0)S;Qd9ko_p;b^%Rf<5D#b`gu-Ph#!eODG%l^it-ckY`e9&Pv1^nTMCqYbU2B!5!5C}%aN>n))!i-A z-K=36n?TJ9)%vC26faaMsGD?(UQhZ1i^e{e3Z$rSX6$w6%#bb@MggipngMhJ<4kv` zrU0!y-(oe3yHeL+lc1Q0XS8PeevpZa14RPpB!uNKf%&0X;vB}j=OI6RoEnu?LpNln zlyCTiUp6`#?{QTwpVUlFL61%UgveIT4g^|02*WDaYx{kTw0w z_$||lQq8JAAo7|_Usx)I)9%zkkX$?>MA^@BR&hWUl-PBaF+arL5t6jMjZu>{ zw*R*ZOOqr%SiZHIj{}6&sg?_yzJ2A=K@Y8< ztkql-;bruvzn)U{dWqTHj;@x&bjnp0wKIrS{3PD1=7(zN3Xx2%EhbyGbwNedXQeJY zm5Uv&Wk`(zWK%3%c0;?^H89}{d1tpd8&*&w$uDGX=3P~c`$Ac6pIslDd7}ls$oG$S zt*h!HPJU31d)4$L^z+fsuVXm608dVi>nE_AM`vO>mjg!MtX^gHsitgx&biO3Sk8Lu zx{G&ysDIumWF{nWqi&XI*;P`Ao(T04&jp5lboO|oTjc5DuHdY@3J)zJr2yKDWn>PT zA%OaDaM-}2m*D9izY~D6-*il-g^*@f&g5+pM|-6_M*Y#vzi*Jj4~Y({BNVU_foM>L zZR&4&;1{m9pH4^FuB59-(4G3ywe{2VgEyI+^zWqPhW2-}an|sc%l%s{BOd<=P zAHl)U@>kGJ8Vwt@(lrT)F0&P#SpH5Y=YVrGZ=~x)?{z_hPzcx}N(dQQw}F$_Til%; z=V?8HyR}27G#iq19#lUZ(_lLKr<6R)VP&2uu__U*G>`dwZP`9)$-MDMo{tRE(Vf!O zU5f3Q(-`Im76V7tKMR!qgftEzwtx!1sG>lGK`(X9j9xiuqRTma!Loh(Rx&P&?yiiR zxRoD!5m4VT#-OLk=yMT|D4qWQog$~%*d8@KicsYUqxfY}CJw2%q9KBq*Y_v%ys$`3wh`lgnW$L4m zo`;w^6<*1;z&N_z{TXkZz*gROVt-yjuH6G%L*lVx3(kR({gLafAASotK8&RclI?5K zgI*BKp-xMz$b(}T{eTmR_;RX}t#w3=+v z1Pdv(ne=^V-(A;0Zjk!<_6=5b@MHofA{0c7MK=x5qD8InmU>gb`?A3^D<$);a_P?K zQLIJ47|k{#T`}=UHG6HIs_ElroRVvQ+qksRxLRN2B{)hJPZH@raxv z_t3!B=W<1Pv~*Es)5-0yA(!A5+#WH8M!Cn+S4C!I!Dm*dcs(knN1UlxuYwJ0_uP0o=& zngFFpZW>trU`nZ+`m>ayIRYWfefM?Xz$OTk^q+C<8}7D7S5PO;KeuwbSo<5c%=SR% z{clC;-1Co0KkfW4G_%q-5r!{S1RY)A3cL(=-pfa%yuATgT~6Bx%5y%o(4QP7zO6u$ zSanLX9mWec=MY6gb=Zx^SfKI3a2q0q$@)G)|jT3dWWEZP?M_)Y3}&ee zaM%8tpVoluQu`W*`v-2PcBa2$p0mesPsP3T}UxM5E11S3r`J{~gt zh8+yfelNJ_84u4@v(BDLq}A&x)-fS%wz!5tEB$PN?0a>5u~ZP-$OP7)_@ z?>xD8&PT8u9ShDCywGdz^Z5IoB@t0~``IoC84D0IH$a`8?35NS0>@c4@7QliF}OxZL`#a4T0N9njMQ-O@M7Xy+t$Lv~o@ zMy)$uDmH3UhF^-!*k-Dp3445tAR8N0!&s&ju1nxcMK#J&MU#(kx8w%q1w|WCkF^~c znq1P1{E}B`U~V>GI?HQNV^lee?r!9sJibdFwythM8eKD9|A;%0+7s*&lLS^$7NIZ$ z86%zdJ?H~Xm(ZY|@yjmS50Y6e2%#B;z^88PC(=&9>I%S^Q4W9L{%Zo^Ka*4*-MYNn zC;$LW4F9JuTK~IwY1E{tWlb%nS$*B+X6l8pbLB8yBsV>jkdZZ^K_)31pD?&)44Nv$ zrD4|PmhtQ~(>mhl=sI(kX)G%2iE7a%4njlC4IuXsFem~-fV+x@3*&-q!)Go>ac^z~ zo`xVjWRA;fYz;;ZsH9qGgUi<{t??9ox+67A7Z;S%l1QG%IF#8L86 z&T2GjH3z{*m3g9n0z8)fBq#+fI;zz!;!9&s2Aq%RyfHoCH3a@8_;vg{^ERUNat5kq z5Duz8<1kcYPv&j_vGzu_#6qjjcVS0%DR%_@X*D_~D;uj_ch-DFBw+Qr>`LrQb+XBa zjQ#9Gc6UfASNbxfZsD9YfIUJAuA7FRMC8kaPyWi-g z!3W7bbHlE1+l?2<*fsmIkFhj&C0yb&>P-c5{-{MmR_-jca~I&T6&@?d$9dk|8N#a} zy5<1;MQ)ndN9fl8S+j68(&Zz)iGnwT1k!mC8oD)V*3AX8PgHWiw#nW~MWCC)4)L(E zIeIchFU28nc84AB zb%Ry94Dh|#HIb(18FLb!Hvcii;5BD6XA5P%qBYW-+2Cn5yEbJM-Z?#Lw{*a*-nLQp z2WEJkE1%XOEgN=g{f^*jFs@1k8v<>Prtvv2xyAk0Y~48r$@$f%6o}TGS(TCDQ&6|& z$WwFlvZ?C%5Cyhs9oiJV&C_Nc8ivai0~%F9J!WE*&MV?Lt}f|q84_PtncPqVm7G-! zzXwIA?@vTvmHMw5Oln~{FgJ4*6C+;Wq1_SMc5Yxi=-Pqx5J|!L|RuPTG+ajP}KL#NU6?QW5b~&H@Mi**02&MB* z^an9*XV>DfT6}qtvl!21Gmb?412I7HgXh+ITmtEURBx7=v3ep1{k-=7V(cAbM2Wt3 z&$eybw!2T;w!2T;Hc#8OZQHhO+qS3wGnsquyfgFWCRHEm!%ixdRQB5I_dL(Kh-!Pz z-)|NTM{aQeHcxEWjGv<&Y|83X~(=3!h9$-SCl)@TkBRnJi~}0Ox;pM#+WT zuq@8^s%NR1jx$6-XRgu(UXCY2y_=|47#I@}Heqe+wjenyXEMad2U6lZ;tYZDxFGjz2n zx4bE{-}j;IGRxxsD5Vx*ja|Od(x#TM?D@bjO7w(m3e<{WI?URxWXI*4mP@uiOm{Z<9EN=7nE4nH zw}sYPq6(FBPGZNH`Z%-RN$?BYCL%VNM?u+9%jY0qNYD4;X8UJNZH<=)Mth7g$M4YY zlKh`J`261OfD1tC;wV5{071FAOrX09v5S4cfCUl6=Iob2de>UnX>7t~3 zOZ|D942T1*$`f{p2CZ1eifve01J4SX_KqPgEza81_9MJ^+7A_RRn>)-?em0>3EVAD zAB?&X1m_DzFYxB0rdR~-L$)9tBr+JQ4;wKmkKg>P^1;g0nXHD#dNt*Q4$_>$1+zFk zVd{58efnAyIZR`w#dm(22c+TfVok}!N49YGpn9n%EP7lz?+;ckV{ER(nD?yOowByN zn=7Y*<_&|PA26%pjW|ExH><2t@EmZHhM44a#t;Ihn0eU{p8 zznka25`QPkrI6W5PJ8L_GxS4aznl63eIX)1jp@rocR56eVZLhBe2ioE0)ZE+i6cDk&cI$(2CiuE%w@(J z>Y<7A3b7Ps(oU1TVT^;Z^?p8}yeiQ{bRP&jGq;;UcG0hUtSB1#OfdBJoMT5QDGQPe z!43k<+R1Gs^d{jcf*Xn5f1TjP7OwiVTWKRO2H-DjrhF&gdqDW_+*%$md2HH@fHZ_x z*_%6kLRD!A!YG7)RluvKEo1o_@S@GCCD;kDSIE`5hGV0S1MLgkv{ z-O^6CJ1_$_qxa&KW7FV7j-v^3bYe6zVjH~0?t*K1Yehwly1Oheqp8Kqy+oD+j~k&u zR}&Pw`D>#|tLXLy$YEv{e)52$ZYzvzrDJLOuKWbkB&B2$K*6et@|W42sKo#+*(C^u z9+<2}!fzS;QM6%^&Bn}!5;U<%10~YQa^dyLepL1VE%WxTBsd{BRX)&5kkuYr=?AwG zYb%cs>5(`Cui0II43QPEKQlh)p#4l8FHK# zb573KOIR0*AfrPi0#kN8u%}N!@CwPX8VJc=d~PXF=AQe~F=Phj@MQ<2s^ski<*ZiA z4Ps~}%tT@E{Ki-L)Dej1K_0UNB-W3X$EsI>w?BV}$e&>;BEpvCnXG|k^CP*jq$pn3kD3 z|20K^EW{5BnHiJg3z)(|4c`(~MSl89!F&tB(Ep0$22aB=G{p%FqNHCAG9i3_2M@r3 z!Pf9BKDM-v8N5bvWFzdo41bOE#+UtDkl!urVw}Ta!_;9P`J&yPd@v=bK$)qI3>Cl& zV&#|)x07qBA@IVTu7`I5JdxQt8#TwtH;~YZh}05Dc!F{_AsT^s!Zo}(JUM&OGk_QV zmU@6`hH{h(!wf|}8en~H$3wK>iQMH;B?z#BvPR@5wqXlnM8S4wtiq7bK2>=YE3lmW zvof^XO8~P0euGtjExmb>OM}Z7`k{CX_uNVo%r1B=2&6zq{HWK) zq^TB|LYt((s-jo?ZfMX-l1^!Xps_PD_HEIZ}(B%qy0V9Ompu$;|_EY$0@0`s1~*o*G8mcrd|0YBI}-@pPdHJDf}tOM@` z_y38ysIvMqa1rlA}1MmwvQC`=CVkOMMjhL(j2d)xd$^0BW99x1X&u+QKgPVPz>9zg)9Z$N z=BC>%O-yGz&>YEtyAJ0YUD=N%4u(o^&YvziQ7wa3()1M(>CC!lziq5|@fX;E4uZ07 zG2ySg7Y3XChmk4YZ4QRjbr4Z>KHtawlc?Yq4~3)3{FbuSfFGkJ;MW>T`f;KW2EKb* zH(N9u3>V3J|TuniLVLlpy|#)t;2Yq&+tWDoe63 zGo#7c`U#Cq+%%Yw8rvRvCySS*hbQ~huJ$E|jgah*@g6|p*?MZUch@g@9n!0RPLzW7 zAQB+E@^j&}tQY4h*xinSDtcYxh~+!iLoe|xdLT1zQ!X7{b#-?nLwjP&=BrmMh0HiE z@LL19sDm#UCuNcV?kI*Jnf1VpjY?G(FKx?3y`p$*E4)3|&svPC6v)TIaPFqW0ru`j z&ntn`#C;!u#OFia7A81F z&Sm;I?N=|r1_Up-cxHwr%o6NZh(P)Ci(>2ccnchaQqbfX1Kx4}-D$bWe?l4Jr^nH) z&m6ajhOb=7r;tB3IAAy(E5Sodf{$C7Av(D&gfzXwJ+M>|reljs7$7?P_fsn0wr~F~ zf6VHhMs5=r;Y*XS`2aUFY#$eyBp!%)__Wbg3MH3zgGhnr9REQ8JL@Xw1qD#5rIE2C z0Kkaf9@@m&Y*}!cdz#;?X+7PfKY)Xs#p#nN%Cs`5zCLzZTzx+)MaRFjc1PUMD@0XL ziIzFfD=ph=lsFvN+yC0l5sy9$u)ecMUb0L<-0O}8F6{_rE@8OK5OXl-G8@OZqmf0L z@X?wpTUJcMCHsinfGQ;}*;hs^u8gyEq9keWb(6I(ChI*hVSk(io&y@eViD!C5`A5yu+P(pj&}?h+G26XE~kjc-(0p;Onjl-*XorEaOn} z0S-2D*;+GEE=4@E$S(RBuaK~?DI;4m0rE-#zfzH=V*g;~&r9_NCXdW0qkaMrg&gaQ zECG-fxmo06FAJ-c_s`N0AmU@&V9t=Yngd}+mWT=p3Lhn}Fx4+sO z@zz6>M2YY<9WB5)l-wa*XwpNVHYB+e@Kh`hvXSydgh+?h((No9@{K47sDcY zYc@VOS-whsCn1HCDeH(LD)XrZOMN<9fX=BNNmOOX3Vv*GzVrY^qy>Pr2 zpiw`5NmCMVpD+|Jd=01dm!4p3X&5^X#|9r8{!;;=F8^)>>v-hYiYF_fFPhEzy>Yd?V4Zb0e4x zZqEnMJLM?8m=iH4xc8$M`DEO$GLqEvDxGyvJQr)Vj$Q%K?QP!_bzHLdF*LgG^!J{q zCz=o>tXJ{*wRliE{Odxb3&!b2D(<^T-NAnF0V`x0Al0qml&|9CUuqQ}yc17{as=+S zF=59xl4@Y=)B$zl75tFlu%w z_8zDveT~&N477X&cP4se2WNFs&m9i!GQC^jSKdClUr&8D4R$_R?^ks)>3&Y@-)Bd` z;8rP@+B)CF=fiC&4cTK$RO^TA)Wo5?eD=dsAOD-je?0u_FU_Qx%;%+0x{~&W1%UO{I$SU7)ccO~qV z7h4uRl6*t_+TGg3n*)d`G9Tb0`^&iZoZ=gAsKl(x1x>(!gzd8y`E`2L>@zJn1RVPt zqtyp-D5gL^%!E-8r2jmQGyTPD`^A=towNpAZ;t@!Y=CJ5>77~bPA1Oj_C93j4-MDQ z+0TCyNb+CZ4z`xYHs&724*y*kNp`G5qVumyBJubBuLl4Bul4^MK_tzf|NB8CZ);`l zX)0wR|0RS3`G16vJfj3mg8naJ-v2oA)Mc%D>tF!@dVc%iWdFtC@ju?#z}Ue_&*}eu z{MGbcVf|`)hwnOkhF}*X$3*k-T)yNBoXmL1!f`bW$LX|(-4P*jTLUPHuY7N= zw*fgiNl5EVxfU4eWPT^$c;EP6SUt>S(1;6S&OEP%L>x?|xS3jKF}lPSR!C1Dr^EUa zZftG44^6vai1vR@4Yy^Jr(xCEqtv}Xl&*T_3A<4D&#iOKx={loQX|&$z=;e$;>0TA z_5OJt1N%@gk|bl?z_o4w1af)EnJ=!Dex|W zC4mR-Pg|6Yj4cH+sQ*U;X-wGh-!VW=|I%WRp>{&=Yno-&wY4>E$^;_$Uoi=`jc}OU zbq`MmJ0}mHMo1v!@emFq$vWvz<`hgE=XwJkPuR5oNTl4KDx=a{pMMv#-82|B>ykl}IE%-UGnh;Gb`+%)^Y;gN)hn{Dq_ef*J| zoftWK+34xDj+e{bt+Tzcp|K97RxXo+g`CxlgoOcEGJcg_(aj>vad<&VW#Y8xENBH} z{d5dJWhx-fB*0AS>?1F-Rhl+b4VaO?Wus_{d5ox`=s-#0bcM(1*Nt$jnh!iTL~V$5 zpF#TS2=<^@czs_@P3FlosW)v|NKR6yE%)kzPNA`ux|@pkHoxBrMd@G6-sAd8nC%{z=H}VnHWn9B=0Pt2 zg-g^+%^V;f20vS@64?{7NcB{K^sOE1rhcx!m*4s16N-Vtuz9rvu{fgqzqixNl9J$n zE^x7^q+tMp=UsI!Y*mEUF%Sv@lh@Wx`h2?0F-hp97+xsQtOeq?PT1oyjZ7niYB(KI ztXFM+ z3j3}KP0!wmnmwa;I{DWACA-2JcuE9<6j@aStRo#5OUKln@9Z4;cbX19ld^mcW^R|B zTygotjED(0%#s;Wqj^**ffA5@rvwp}ytpD`R$!AgGLJ-c&Mu7TsoYkHl(_)6D|XwK z{W&DDmCzD|hO0Mnec-=viJLemYCU)44gqBd0nY1|AA=4-V-@*5(128pMK zU08HDC;Y(oPRO@AHVg^xy#;dd5%BBE5Lyv*O3N*h?^m%EYD&FO!0$z3_ZtHOVAnBu z2r~A(70&FDA`phMN5~I9iCDb5`avTha#_zB!#~ZTXT={E6^A%z(T#w)VgR5X4O@r? zEt09(1f?KWI^o2C6OE$m%X`abKEj}X43UZ)Mfj0?wOqyGSMIE|J*PhDhb*9Hqo<;r z`@E9`?^2Cr^zyPsrvVMWm@oQ+rkE@zft&Jp2}L>L!LJPy_{0fhk$65f-gFQgS5~0! zSe#pcys9i%Va8GBJ0Y(lfG;@>4jE#W$R^_eJA_r()=4E3csR#3Sge>081#)Kjg6&1 zhj>^{*5haKghc_Fju=QO``2dOQ2nwSS!)$cgCLk?N@(_|N{@oVE7S&f6-jq`j~@!l z>~H9}Ui#GB2aZL5Ahj7ewf-p(XiUVQEMJ**po1A>Q60E(g5ZsE`STR-id^qRZA<@$ zEFMEhEg91YZ5$<#1sMZ?9f^hPnmwLKvnlEtLE0xw-3A8&hLaL`#%+wiw}_&v){=;F z^v#62#ST6Nq71t_>Lxag^e2Tz_~giVi*MYKjiAV_LL7H{f+B8nBW&-_g-gs1f>R;AK9Jup$tootbRdt*~YFv$c7v=SwbE58GNLq~6v=z6H%9HG4zmohsr<%}u)%atl5cVlLq zMa!Qo9tETGnj=QVh%MO=E6gNzM-y*h^Eb~@a~)GVfH5#U$1H_l1uGEn1zH%LEmbsi z7}}(|Rb8zbY7V1np2~sZNqUf>U3d;^6_AbDmZ^5^OROkT?k0I1x6 zdKbnR8PRT&$TyLl2{rG~iDy8TG=k0}^+i%~NJ7dBqRR#veftPJI6uarthgybdb+%F zYlHD;o#iIBDul)ca2;_Zf=Ei08p(%waeb|eFK?}W9@;pxzxPTh=K6sEp-#2gg=MG` zJtxGtOa*HeQx%IFZ`r&>4c7ZKy7!SK%zJWFRevR%rFhF$jH00+2N@GyTXx`YL+zqC zCuUbwb9be4v^>HQSmfOI2jbg=hGWb=iPmjsAs;W_BTdS%{l;|&o-PZfGc0)L7|?a7 zMtl`m=pqwe-wpFk$f{)q6cg=YpY|f&;>j78LWSlGR_{N!jPa_pR<$;WqExNaDed(N zT`12>rlR>M{bW1O4k+uXJg-56Gs9s&G4;}$CC*mN>eN;CMkzwkD6~kLdRwE`L^T=K z55>1>g~;`FsG5ssq`d0&zdk*muV>Lb(h{tTY~mpZF^IbuR9UB-Q%A?sDlILs(w7(B z%?lQC^w8+-*zp593e}_@vU+nV?;lyRY()KbM z%7*dUzugY=&UGLs{MnM(O(s zgL591E4=PBF_lV)rN(-HyoC>23+1>YobPbq)7Xf^sIxEywP$j$<0|F5er*+4;=wHd zgGg^^=iTeIxo0-SAncvpD7BH15g{+^y;P__q{9oQu#u)AH8&%xTiZ6dm(e-+eE*nz z<_6ur_RoMhYV>$4_d7S;Ax@5LfKPVKq&c=AMryDeaAWZ0SFi6TjuoOu2$tn~vfc89?gBx%G{s~e5dzk#5es8HpNF z)ppn9UUhIf^u~@?_I5?aD^3&^pNRFJWGBz@0_AsRUj=Us5DdXB>DN0~?;1UXxATr> zWPePQ{U4@?${&3i&N-bVY;Joj*fRHcF+yu%O+H&+fs?>TU4<-kZ?rIg;qEg8CVe2=J%hp0K9rTcn8c9+xp3`kyK{Ot5J zZ0P1B&+ylst5AhBT~}uDrmDxlYo=y*pJl7ctg^i4u`iW61+14#BG)}={W%L6Eshz> z6KEV#Mq0mhp%z{j*4GsGoBnob`)4TRts4T1ke;3Jgy!Ar>wZH_SsC|k?DgbZ4@}6l z+vBoVGn--`yOg?pm7-=eXnu6Ghp2Zg(84avLGtONM$DNbJskAXNi8x+N`Xk=VM{Gq z>yN2GSd09iO{u1Rs+GD3@0HxDodVzVTJ~^wi4XB!w1O{QUK4MWU!Q)jtc~>I zxG^KWs+v+ypUEt|e*QY0pDttZG!_HKSx#3Vj=q393jVm0^>$j;;ACFe3|u4A1Tm}W zo=<4aE$>E6#FOgv8Syne?6yNPf@+YdCy~(~pOwgEao(&!V?kdP^mIDm@%pM_GtiD! z;DR46Iih=-t`v1Pe0aIV#Q9d7Wb&>%enbC1MNrh2)^-$#002x(0089w+lM~GB~2YG zTn+C(%r76uTrQcbVQYz#WE#uRVaW9p^?|h+m0f<<_0^c<7@4VeuhW|s_Q%uOGhzm| z)(BHXc~Br!RQrICgaRFMRmx?w^eSbUo&ZKj)utkNm{5{ ze0+M2$CEz?FFzb5k2{7_E!$rmP0jLuNZ&iUtHay`k^2!1k|bp~+HYCs7KX~{SQ@1^ z>B5%cty0=_&?=1Asv&xaRH+4*Dh!&^mn;pM$TG{_GSF(EONNtz?asB$p-TFai2KbQ zibykwwK0)pBdEoKn8`Nn=kh>f@=x**a`dtgrrnmg<6iKPr|T+RnQpRBrtd0WgttAl zIg~#Ak^9q*#}n;j9gMNG^s;$H7Q;$PTT!bP1pWaUY|h)%V}hwgGAODXys4p?X81!2cs{$l{>-1` z$jwSfL+=m5tykm7=7Vsrt;%esy6(5WqGf+*Y{+e`>!>XS*-3(C6Lv-auSsK4*%$l++lFu%iJ_azo#t_H#HA#c{a&(&X zv=%3J)ZC@ZR7$}A(_KrP73Q-s1IY}f(E_7mnYSsnOwSq_$^tYUXtI*AjglM-TH zNSIE;cj&~YJ^7!e(E!OwpGNDu5;TRer>@pPaxwy9m;i$gjvuC48x?A&hN6iEv8Pmbz907n1PhPW)gQ(wT~>oic*&5%HI|hrdfdsdAr}H8k&#iHZ~({1pwzqTs3BdU=}VdS%GXobNF{=L z6h+G!%bW?qhs7uL8Yln{fLDvMye}sK zg>usJ?8oG???NB8RHE|^ZnmNgT2lFfQpa zoNi)DXD&&N?M0;e=TnO>w#G9jfDkTkRN~Kwz4&e0SRsV%c2WS}Ev$nWfhV`)n5PPV z17aj)QlV;oND6)>vO^@D;y*Ckb6R+wSB_Wl@%7sCd=(^7Q_y!q^RH3yjz;cIwdMj* zN&2Z~__gYSLUJPFF(d~RuFJI_#>3EO%PA<~v zAA#x`Xnf&Vjty|ifQ9#!6G54rF%z*%K}+tnu#hT1Tc-TYcga2i%mSBm*yX^%U9Cp) z!fn=48yx>FGF)gSGafqS1&o@dkUb3gXN=m|^(Gs6{8B~%@&LwJm@*?5VAoy_Ax*kL z;wD+MAH9QpN*)7R`Qi@R^JjOZo?zCC?9Sd?qrh6FE1yN~Ft8szub$fWx_sS8W=YwxV!W^BvWD`shz!a7?XWzw8$Nftyr7{%!YzBI-Aqx4w+u`bDfzmp z*xxmcezSst;AGO63>^Ulj^TcEtrITXu{>$Prc?vR&GDKUaQ6hHX+1-Cclgy-tj?X# zFyoC1$|#Goy)K`nmc!L_?W!4@+`BShWplJ z!^{-4rwS9YaTHa|!09s_8qXBK!u;^X0nU1ic?m^p4#|R-wr>s}E5i(DJ~z<>05@iS z!Xrh1w+z|6?!Q+O#coMStmknJxR>HpzJY!PhHWh8Dg*|rBo!bHu+2MnKVlFf&N}t^ z%~RpOHvm1HI*o3QegPNk0f7>#V-Qu-(m338oE}j9rKTUkFfJV%F8eM6W+_wMZkO)sYXNcOy7FJDB9pc{H|o|7>a*U>FndP z!#w0dPdiF)$}T-1%^k2p*>W`qd{`G`+NA9eBO_cE?NX3mHG8L#Q#ApVOb-Jv@`A!5 z@QbXi{?MwnXO8O;h`INGO_CSJzP-`M7JLTK(8J^FV@e`pizpF4{1wtp4tv#GfHysu zV(;_Q9LC8Jr>EtBbLW)!>t3g#p1?V2vm>(+qce8#UTdI=l41Ca!Yk`L2w9iK@a-3* z3>Fa(#sesaDPXZRsBqDte)}c)r{a^lh*biEtT9-(2uaXh`A=zH=vES6wUqZ~=KP2s zlcZ9w8@6{etAg5h6B)K7ZHQ>|$lxpi(?hhfK~aNOKA7uu@WJ3vW(jT(<&cd_t>t9T zcQ7E(##JC+vY#eX{EWO*WJGjh`Cu0j%O70zlcX@8jdjOz#tt>aIm&8efcepwHr@U232l1xck0a_%4#*}FV<_*Xs#zg-+&L- z)@_MOPpdAobn7-1dGAY;xuUJx4-|w|o4so{F$r zz!vHzceD1w@f%|0W^JZqlp7n=zgaX8d z1S*y0@aA?NK=Nsq_0FjI_;3QV{d! zZG-8&yV7x$Pz?up4n4F+ROnGrk{@8OarmP@V>|Q=EV&c)u|dS}czRr10y73SF%=+g zpPf+WuI>oKizbxQVO}{$22B%h`{sAEKi5H)+f8rOagrtQH_w$=K((lqj_FP0($T`7 z=3=oUvzPBLm4%c9mJk&}O>Hd)1XPY+`}M}*Adefr=J^fn0g1R#Vii#6xHgTOjsV;$ z(%pOIAF)2!+#FPNJIGwVA*WTkLA+Yw&Dl*n^qmi8W5ZM^9aqTUzV+F)kW)~t$=ihH~z zkVu-!&41@%`&LENkgqfVYAez{}S76lRi>!j(P(XJaojF-Z`ShYSi9rG6 zYu_+8nnSpTRXw69kEY=Bg^~x+A|cZVuKY>o<{gX=*cD1*-NmbU7GIudsWjq<* z-M@K5Py;-Oeso31CD|1KHVhAul{Q{Let}ihz(#Q4PIloYx~v#ixr4T* zG0cB&WarwAu=y6Wd#y>>`T*UxRV8S5g6`k(P0(7v1ym#a{=9}N<=%Q!TNL#zIn6Jc zDB*hKT3eE;O&Mq+2Mlb_b+)HJ5%B40XU(f$^>RXCS3e!86>mPdB-E%+trny>-Oxd7 zgcA+KVuud{nP6svLGK5R6EKBu+2%r?9mrspw^y-E(j2SeWqy%uRW zk}TWsZ{?QegS}8vEGPRX)u(OFqR|KVe>PkGL)aTX+gLE+001yJ{+nitv4f4d@qa<= zUrL+9Pp;qLS96g!SRA{T?*YM1(->rflqN9IENrEA@2!*|`p2WQl*d1hM&$M(+%Id(>5-2jJ1kptUGY0_B1uzho-}l;FcT)AGjs`x-+#L~ z(L>gX1`#3P%Rpifc?vKc*bB&22|e0g2Izowg==kRO@l5)NVXsDz^NG-i@)h%VQg>i z>}`$>LYm?xU!Gz}@oq-CxOQYpS*lI#)50;=)Rd|$c_fINDPx~y3*T?CcSk7K5i75EXism4cZ&!}dDL1&SBdcZfLNp@ajlW*pi&5is3?9a z#O_Q<{AJWBCO%{Bv#^H-kPPD#S%kF~{?7vzbMV0fvqByAz9C4-6>Z>a(sIeb~I`g-%h!e;p)6;KUE~LWSsgjlF!N+s_P`T{Hz) zC^e8Z0{<&|=#Tg(Te*ube&I5Gy;q7^!Y(C-Lb}M|pgP*XPCfEyd|@d}wBL7mJmcgM)7 zB&Yrlcdz{qcOQ&JaF8IRTI<0NfHg_v*%xQXq5eA-*K*tMSGkRVsL$vzgu+UTv0fL1 z>EoO|v(MABA`(2DeShS8C*sSO0%?fn*21_5>-~QAH%*CaO70fRI(plfP)@BF4(KOz z&z@imaMvDj3^E?K71Q*UEQp21UkzZ3T`D&ok~wj}dBTlU-1`>d@%3{KzyYi_V!SEe zQDn^kI^HW#5J0r~A5EDE{;tIb@S9_vc)Tc`&?DhF*BLyR5GZQs^$VYw=|LrC5CP#F zX6?;dsU&_4q)|)8xKh**s2yqL#8w5BNU7L$c#5YQD;hYsm2gKygjJDbkK-6MLg9#% zhi?!L(H6N4*!VgX;F98}(b4r!a$FVZ+%Je{kC_6s*dX+|)` zisun`G++F;5prmM5-L+B4-C~rzbdRR+$J1b5&j8Qu27Im(%MBJ-3|3HOE+NPTD980 zJPO2EXs+u|XWOMM(C6)B*_(Xp00*rozX>g*{H;n~zHJgKMW!uKrUfJvMCF!94pOgT z&6^?gMm^;tk23xwOL^VP<7f?932VvZbv+2*0MQ|0(gw=pE@})713(i7W$Ka{lvh427=vQbzuE!9P68)?4I(5l62tC6ufA*Ya^0TzLchUBrJ#%15appVaK2 zmSBWgB0v;m2q%8+{oP0+agURj4=dS(8f>(9oQho6cl8iX+tpbPqKK??FS3HhX2q<4 zu4Rl|BbjH+3y1n?I17_o9oqJSUU(&ld*Rq8McXI_RDr~fQ0cg?*dr@k>R1W}I*=C$ z{t|Bs26iA4;+XyY$9?W6@=Pg_xa%>3>)%s%`bP@ui{_|RlWPK46L6Goa4a@)YPm>2JxH z-mN$l+X{pr!&+^=rND`fbbb>AQr0L?Zfgd4mitN}#E`t@5}9-zjoMbxq%MQKtPqo% zw=CVifW1^-PJo?tealmK@)=E|lr-_6Au>dH7iT(;ttG22uLVFukd?)0&lj(JRhoOs z4tWemq(Q#CEhvb})ZEh=b{(0@qfp0R+Y$k6atPgeHP_-V6zOSIzSm>hxQHPO6ZKf$ zA~WJ%EuA(XxBl?)rz(q`6?;!^9CPjNZB3gtz60Qk0DqF}kL&|0BHu{Zqo+GDGNQQb z1oNeGr&&RWeK|o=O=B;(n)%M&W;@T0{G&Xb%FY!>QUlYti)t%Ab~0{#Q?||eKG76| zmUW7i*G(W3ba3=qfx+5AH$7366+%s}%07E#ly^ocXtt?=Z&(~?-BsH^NPc;!n==}IFYgjp;elY+Z~HFJeLbualKq(3@q|bl;?%!9aI`0x)B_ z3RxpDxM;AHyNX%^dD02ZP-#`!ZMh9S;(7U*)5Xz91t)WZgR(*f0j(g9@IOOwihM)X zlnqa{t7So^I$QWv-2#h_Ntsg{elTA#!l+Xn&f`VIza3TgS3eeGBF zR>I?l5;5Y*=j%?PIm1}c zWTEX<4!K~mJKFqyX!ZE+{pPglxLwgo6zEywCKH5M|9Arwm|;>8^&n^-L6Qt=rO+P0 zd0vCw8KjTBh*(toXkr&A6BTmJ_OIs;l_}7eJ_(fg4iMS<`E}d;un&<7{w0^@d(bEq zmku5{w1gAoY^W0LaahAis|1Cdm4>6{rH3obT5{DFdmv~`?}izUtt6MS8=#?_Q71UDF{3f$!`Ep#O4oV4Q1itsphOyH^R=gRzQ~3r5k7 z4%WoE*Ru*wyWyF04*DG4Zt#<%26E@j6VI1D9{eRcxR~LzO71H1#hUBi zI-qC!AQ(C|d?TBEIM@ccs4WZCXgOW zphM2Zum;YKYb>L+8=~_2s4~PpZ|dR9HjJdUrGOIgEE&5Mh~4ak`p@!bnW^9{yxR2Y z5bvJt&kjLxZ+)%1dXq8D!1i*I@r>JNp8(Bx$rLfUhc-JI8Mn0T>0Rd+XSY4SK0n~O zL^dhjUzFbbuf%?lrGn}grEhzn3g72iyi4?NfLA8#{-Shj?$yjKmwuJ~j11oAs;(On zvs`L1#H~hZEOjA_nqxm-pS=8*I$W!`k-dMcOcZT8>HVTuGIt)|MP(>!)w`>6fWXNA zHcGi8;&z!i`r8w=@pDn` z)kopp8__Uw$!`Zw)()Fhek?I-jy%NATPJMG>8H z%T*2kBD>>pDOUG^4U0~D`T9fZ)A5pK^i3AJVuJD43WJFU-S<1gk`d4QNKL_y^O>0} zHMtz+bSdb2gBd{gyTzex?6$-gQ8{o+wl^i(lS^V+(=B(|zx?>DhL_o?-dRci^SZCB zRNdqJ4UWvy!ccK=%`VNtP8gzWfSsJQXyoRy`HJmObYusuh*=<8bCR5b+j53q<_fl` z=e0RrXnZ1ObZNMxwPB01;{Idu+`olNsfeyvm@jH55q-9p zudGq0B2wr4_wZeGLP{f%vSDIxnuhbx&V&;^%xMfLDl-Jge1V034v12dpbLa{mZxb- zY7Y8+Hs!d;mw&FL8^!xf3OQH2XUBDVayq;+9%DNft#80Fjni@3_02oQQTe?lZa8G; zC!&b$%$M}Lt6Mw&)j434Q1ft}V1#5RS0({XhG~{gqC+@>bcs%?Lps8Esam{4FamYS z=l+Z8O=TS?e~54ikElm9f_46hi*~ueBLEHiaPgE_caqTIToIG3gfI!w0Z!tGfpb6+ zlhjn0%J2{;c^uI}bc|OLH`@+|HzsUbk4JPmuJXlf+cPuMP~+yV(={{muF{q0CX3)C z&}$cjGeI$p#8i?>=`b^KT;lSt&pWg9vQuV6$6heD8~%D2I+*H4%fV`tDUZBQI;89v z1^Tlj&b#@3bLW%|2B}5vaL@toqk)gY6V7oyw!6Umd7%LJ*vH82|H0Th1&a~|*@6f6 z*tTukwr$(CZQHhO+qUgS*}hrkqe+84sP z-KOv5Z%5<@&3@r(!8w-*iBu{6xQG1lW^l|%cT}XZ+&gV`hi>+ahaUVL-aem`l2xfR zy7sCnw9eL}H`}!rHacQS^PTfRERfu-$t|XxoMUpiWmV_1fYhH`vxk_Zw?qv#(HO?x z*bF7mOC$3vrBh#muc-j=gMV!=zimFVHuf^a?z)L=nCS_+-)2Pjps3QUVg-1u{MHjt z{E#6M5l_KZtU(Ln=H>xwQ+o@VUDz^L{o~p{S_qy4_xR zT&HCXIZuz;-6)%~(}Yj-)>~8+c4AzrV@4E)oFUsc?twqsJVd5~=p%~psVU?-5(4?1 zW`eo=5^y&i7WLd5#Ftl}wbKYvpE=_WL~9u*9$7#*E*UI4##)K(GU(`lq~Zo@E1lLB zfBd}J;`_Q;H~jrmsJla0rj{0y%Q$mAGtW@t4F?qB>UTDyOGy6~d;pE@;07-nMDQ-P zlR$alILU!K!(bhaCO=>tkxyw3u^T~Sk;SxD5Kx8HptUzj5-OGrOm37Q#S^Diy991C z9F?U{jIM&y*q?Qm%X@a1*I84!wkeusTZQ0U1ys}SK4-g6NE$6Xv8Xlpi4t%kFWxrN zxRo}CyIvLcp3Y%vs&gL8mq#-Ql?nTRRe^7hIjO{&Bol;+Gs_LW4d5=k-%pa}7M zX-*X8PdcfWX4r_i7LqfB!$iMs(0GHKMoIKW&*`H)HLTG=9)ZAV@kpFSfb~HA{AtZ+ zJwkRhIM+enpyd^73~_NVel4CwoMYtf)8dx_BuH4fTGyF0wHb-+X?-@2DWSaGq^_~q zYZ*CQ2h<`F0JQP5uDbG+%ndkVifka0|93(S#j%iu8&AhRitv=08C$(i2wlCSw0fye z@ULl8+Uv{e$paHsKb4ukhU({kIfsFWl6I7$C&$AG_gJs)->B=C&nLVEq!e1t&O?p* zal(MmFqtDe@c1#Bh~#}DMo+g$YtBAtbd{URWG9+VPbJV(t~Tm24wP&1tp3w9a^=rM z1k8h?_#-u{c!525c?QCyj4Zz>&KgITOWFYcwZGWDE(BZ-dS$%GCbL=jw~C{v^uZG7 zV_I99ekC*+4y<3&V2x$H!?k_G=fr??%LQ?k!(a*yuS22AX6zq@EIs#hgV)%4FgY^J z0ZB2tNz;%P;d+;Cq)G>zsO+l4rtP&nTzPfJOFIF`I9jJ~DY$L5n;E}ibXJpw?mkhK zY637#r$EWN^iMsdGQI4bqHVTm%WNH%W~pf!?V!939~p2dbn}}1aqF=R8{Z!sq0`fs$kpW)QFJb*Y55N%-pO@^cM>fw^!k-DN0GAEX5O7 z$0d>5Na^q0Sb9V+YR7&M5jQ-dbR>eyYg~Tm;P~iXqO}PO z?zA~MRDl~*ij;q{qExTsD##~xTfAh74^(tg08HS&k?z~NUZeux0lvjo%`iH4HdQXf zX9X0iuad}g=`MW(CAXOqASh_oS#hgabni%x(i)>Oo%(AXuAebfjjBjQMNSfNttO_c zB_=@(Ze(-Uykw%aGyH5SY;)&16hsWy3nSN%!)PY5SpDDFJ!Q=tQv;g3yP)?my?KOx zVEt7A8hO&%V`)Pw{a;lkGtMzb2Sa7Za+dn92!M$^gHZ{E69c1WR!YF^qD>Yz~}B#U|O%Y-*5AW?iL=rYZO!;Ry)|CJ(iE0FITdW*1F&UHT+YHyV2W>NFQY zwUBlS>s{{inQoOoZJY}29eE3f7(2N#4ZCeY%I7K*<<)b{|6qF(O^8ABrI!r&!=u)l zVI7zv9ESRtHS@j{pHydO@PT-1_&z4^T>V17{%E7Yq@uwX1z>HCLU#o!%gendi%N)C z|9e3@`(nYZE_{1K%Su~#kv%~i3K`;hsd24G{$`*Ypy#UQ%49hiq-?OWLK~iLN(>hw zcs>yA8viP#OOfdN;px#d4-Boj`N7KLRkIDM9W0=0p(_%DpQ-DV9|Vgy%#CV~a%pQ6 z1f7%okM-)tJ-$P~L?AV5+{yMMrkj+KY(gL8t-pt=2C@bX?&@tee6A=>R^OHy3*NCL zScejW1hVOOHT^!zf`9ps3*K3AM~Q;OE!gZHg@2=gGvmh%d=Ey2~fgj zzBwbmB3H*vN)ldML|>K_0y_C^{xKdGmV2uUFf|!IMO#J7?cgyS#3~wAHLD3iyc~xd z4h;(zVh59nNVwJ;YCeT0;JI$|ht{IRZX@X{S4 zNZ(kd+?3J*j<842QGTP-M5DfVM|tASNfm7h1?GyE(1D;Meq_&yPkPCo9rJn#KYEFt zE%SWD--AkJs^@i5zpcwyAwNQkzgmiTv`=y3zi!38?elz;-@oL~3{OrxfIvYlV*Q%I zL~X0vFQDmFUs+mi)O4tQp!i!bC(6x3om+Eeai{rRe*W^$jA(wJt$%7rJ$^sPwCD32 zA1?Kr9B@W1?i>sX-fmwRK7hMFg}NX1(7y{RbVwB3j>jQ$mwYgz&zk#@zR66vV?D5v zME}9Z05aa&OT#iH5x2ZD)KT8)_u@`O6VJp_ry_MoR~q<7 z0~=Y|+YUvrT7Bt0^I_INIk-X9<2%qDaR&l4HOJqdVikwr34Z8oAv|5L!`Ot)u9>I@Dg7j ziBYjC=eooUcW4C2oUE`G(L@AMX@ zYFoMA?`~&F;d1x8y834d?O0F-D|MC1I^9e@h)gM8P@(8cX^O#uA4HW5gl^gZ@i{N; z+DbSVd*7?36;Y&Xr&RiKcs0EPjDP_bvZ?df9zzsx{)U_4EpF0$`plijB{IUH`zod8 zhEVw#OMz(s1Jq62meFaDMyzB@~l};T-6(pTzoroDoea{N+z7OXfmk z0iQwnnO?nH9JJMmR*orT2x7@qqnv}DkiDFAC^xv!()yy`O1>He+!9`^e*q@}ijPj; z9(i{=qHxIRWC!#^Xe<~#)aCYZi@8z2obswlxZ>o(_6Mu7&CcFyPI}pVj~(01G5NfH z_L|rvC48zK9O?Zd#A|Q|?ziG~4jSTx-4nM9OEk69%GTApm4^cF(AsO9BLUpCG|-dI zKby@nHFp7-tLrH_3Ao&Dxl>=yuUAEF3ypHmWg0Qo0+&+~YliG| z*h2q|V>RP@{xTEp_u-k-a`@L;XWQX8ZEZb3OQ$LIx6V7fmt&=`7YjAp@t%gyWhym+ zj|^0#I$YS!Z0_=k#?cERzKrm}O`Pkc@O3hjd%U{u?bpd5U)wVT0ccAPlpFi9jj1bk|>{3R{20F^HAE3uMZfc}QWsWsF5ao_t0BF;r4dWvyd zwQFw0gONPJOHGlr7#jgXCpv`#5zq^GL@NWEl1`aBy#r9eC4tNnN_oVBI)!eMrPF}+ zb;-3huJsJEnP=^v7X0nNlQ!LjRN?~Ge4Ph(Np?sEo{-|r2am*@ujHf&DoMuO3xeVW zZZo~_@IAHh561knPwu&ITY_(Nw8eB2dyAL7Y=lI(LiFo~G7&C}LDHwF=buBXMg1388%Meuom2wYfDo?^nD zD{}w(UB*Z~frqN%ogXgBERQb87<;M3jZM>$ z1*c08wnBaS);v_O8X>x;0=CpoyT6Y=uPa%c7#~U{Du1^&;y2&9F$6-4|DYryD2t2A z6a0@J=F8|X42%W&%DIP$jx0RFeY*XM<`B23)dRCYsJxoYOgeIjvRSL!n6MQ?U2OAW zl+$pavXfGNDvA5U}fX`q&A>4@rTxNSvjw>pq?dwKwQK>jh&-0z4D~0(Unr_6F8AbS};o)=vLl5(-DKow9#gP8Qa{Q)`$m^>NuRE z01+*))-{$Bm9*MRpX z>`Z_$PZ{98mZ+wG%b;MWTT@}2PjXb5_*GlWZNze}KT8Xo9)(KW-cHbz<^T14XzI!Y zii2m`_S^31C&jDX?d9g+!R6&7RJ)s?rj}M`*49)d<^GYYt^FhZ!_wu=LkcYoGE`4j zBK!rAScK7XH4bBh>)%1|uRNj1dqy!`rM~PuFfO>wq|xszK5-mLY^f*N(|0F9y!lN;+Sr3nETpb@|AmJ{&`+=N6Cr1s z3o))2VK|Fc(EgcGqTG;H8Lm}2e083GOT2&+H(AYp`*-WW?moCG1Y{C*d98bWKi`W^@aRzS=|`;KxVwnWjc03{Su~& zYm;@7nC_TDRx6sLLY=z8H@=9@M@ARXk#piW1K`zJeW=uBCpWb-O;4Csr4O@#Md$rHN`G3glH%dLFhjRR{tpF zV*tqSZwu+HT)ANj`-sH&4f>}L>I5Pa52SO0T}*j`gg|&v%NR<^xz9qeR#+-?mD8Fj zyCFvuqX4%!kx%Epfn;@~-_@88Xi?5SqMVVZqx4M zu@jV6sa8IhKBss1CLX2ig{kpEu*Pn-1dA>@=aFi@Bz2sVm6HR5c8dD03kBM)pD5rk z6lB?_PxYk*>Bf^z)*S~g!5=iYRh-p4p}sW$-vz>J@HUoRf>eU=RsDh*W_P};unp4k zih*z~Y!`iqu1>4&Hx@rV*}!YSG61=iY8#8Jno$sm1Uezc3-9x$VG=!&_KKA<4 zHblq@1X?4*d+up_77Q#f74NOMY zZDlD1=16`zP)cR{bBV{9@cDN$j^*zG}qT@3YwyIwaT3%{{wmRvNahu<9c zu)~6myW!9Enycrlpf+B#HKwJ``M^sMo@eSHF|>A6ud!)UwkOEJH#6({{3Z>k_l@SK z3iW_W>p+-;;7QK_SDY**>LTd!4}dvtUhmWu+~0EmJS&Y?58bUlxrP`n(0kKu9j5sy z`xZEL@X*qo4NBJ!E@H~1MvXMcyLUX%R<)?xt<|}Y@zz`kPSblG99nEfx$J|J=$$*d zo{u(o{6buC){G6<`8}GCb_%XJjz07^-F?ni4V)*vgUE$IETx8nz4qsr^Rpr9W5vOP zV3aOo+E-YsOkuXghI}SJtJT;pi*$N22uKQiT+>DS#rxr>ryL7jum~DetQ*| z>ahv2B)hbI!g)DIQf&$6_;BWeAE~51v72kjxQhhXo4D87Utxyxam_bO+p6Ms4qxE& zM28Lbf!efIYG9jP<3o=8%XUTz?l-|`JVE=^1z@Fq_c_pzF1 zvv6g^%F)>kvpguRg$weoU^W&@s~n0tUPaHtfYfR^l>^b&vOhTM0nkqezu9PKTcm%P z8hBpCoo6gJ3uL?XuJ zxVBuYBfu;d!k~YFc{n-!nozIhcIqAJ_N(0$nrteSa=vVR%;lDJ)t_GhB4L;6%~-)E zK-=EFGq)@~iai@{%}QjCbyZ2z-{Fc&joy2)KWfnGaKVa)#eQeV>ga4N(06%{I3AvO zYlZh)ilm!!wvHx)S68V}+FWif31O7gU%LtMDsrxqhLsw?yojDwbJrchNhuP+AW(;6=|5}e6+w1E6Lm^bdJNl6ff&HO_=P41@Oa>g&GCZ=S~wr z4hL~l#oa$&GEAikA&(!3M{ER7hxx%A*$KR9wB>)34bQC>Y*G}r9;I*}Zq7$uu^r6p zPLn6O%5Jn*K3`=jBwTt2eI?GZgP13%EwdY3zeD+BMvVH+YFoRtlPYa3;&*FpZO5G- zt&GbJ34QJdX(2C7w+WWnK%qws0Mu zalBtOq3!ur_WD}=f`LTVyZRMhIkYbUDjW|F9Msqt>F5mO@wbDW?E2|$M0j(FhA7*%rXOO4|_#@FcXy+=SU!S6(mcHfGaSKfqE=O5fiS$)RFjbDQ zj^jo4PXQ-*z%prG-j<=D*ep|I9~Z?6JpOZZp0CQPscV z2?h9zs|iHZrLxq|$4reySxJ&w1v3K93y8=p25%zQ%g!^5C(MT6rm0No{%vSTS39NmxsTh2cv@gsrj&GvOUY=VuEg9@N8q2z;9v zs0u@Gawt8|gqdx;vPAB>!%gSc*#X}cs@7t)yB87-zxlxJ&^!Rr+(U zlR~<-RE+fsIXe*A3pM+0R4EP^O2CPj1rG&yhH?)}sX`A!$%2iS!dV_Y8GL`&eA%qq zBQcUJmeCEy zZz30NLy420cL`|s0DI(E6OzG!iwiv@J<%Oe8Hyw9K(>Zdxt??F=Cr>N4LjsrHS$Ne zpjT+*l>An#DodgIFYAqSb64Z%Xv+R1o2p4&8z*#8IbE26Up?pKQLiS8eXFq_dDF{1 zK@unUA@OXGJ_fyxC_^43S9*OJAub}EUBq_oWmDB*0+pkfb(i6s$Xrq6IF(^tV9i~Z zppV=!%e--yZR-49S*$vl^w)jS^KrC2Hd!7(gSoo^94X5KfeOV>v?(O!o;i?!5*-}JKk|!TIz>J(_(hJ z^l{+zVT>FZ>thnKeU|y}8pz+OUE9&xpSqqVqUiD-niOa<)qiTim-hfQIG##Z+OgstlmfZZE)<*SaCb z=oa}ps?dFV?N{2hi*CoEOl?+ZqrVH$B`w_)E2+`7xGP+o;$y&&x_qW6tW z=jIqIoe-=<;Odk?{97=Vvi3}pTmt(>xK6!(kRukzXR!7|4HIbdhk&TC@^#BQq9qJ6 zeUiMuH}TJ+z>5@j#uw&q#XX+7$=54KL8MXB2yazNES_!qwP+wvXRKc0!|d{D!#aaB z87Qadr2jO?6c`eebEt6z)hxUm5s_mG@tL!rJ=aTGg|u<1Xl{+SlHbp0*x>t*Mo7kA zR3+-`{kYMwKY+2nNn>vX>-Vv9g560_p12nUPX%Y2g>x3nBiz%YzjaYkf2MV1zECLm zL=|&JCQUR9Um}dzu%De&@85?CR-e{!we0UgaYLmHO%vsY*bgJJEH=$xRXkoWF@Hp$ zS>OyZ@Tw9bRrF2q(4X`~9VQ69&H!oHb-(^%%&67^A5#~3ELgBB8N*)A1{YEM07cg2 z>kDZ5uqFambbzY6o9+-qSeQq6WV*RYcw5}X28Z>Km?zL3OlfSQukmAu%`$wkS%}=- zRfMJ6Z3WcHP;#5eg}eaT=orv0Q;*Zi4O6%2p~PO)H2pFKpcgK00Sr+1=hp`SfhwB{ z$L*`&a2chp%@Bx_*KiH=aI&Jm<{pwc*~?$@3^}2RG%Mfw2B{-4uD)}|vX5>Ex&|N0X+x{S_g3%M-sBQf z>XC-&*kBxs0>i`O@?oe4%%S%rVH-|#i1jMQ7nR%-l3F{JFAUaGF&)HH7JfqZEq>>~ z5Vducj<0Y=ac@KND0Uf8IWZKtCpqtdKT-dlAVs7vKEDldk>HcfYWB3T6M+f_5 z)k>GttH?Kf1TP8ytUqUCj$R6Z5%I;-?^DX#*Gc9Z0%ni)qoD_Afig`fWGP9|fA0EX z31_=+01S8M^rq^*7uwz8i%u8~q~-l5m3e+VAReQG3?M24m!PzXNxDK0BfC@*K~)Ic zSI#QkudkH%kA?X#6Ok_4^)WupH|PVcf<^iS>{#CjAg!0hqTl(#(&4yQJqc1g(>bhO zPhhw(>4B-9sLd&}CU$~TUPYQeY~)7d_erx$^eDCvk|oLp%82k)Sr2ZzldpE!11|vy zWBGbS7%0j%_ph>Yz1hjZc_!ZNlr@uzqQFF*oy4YasKo_2UaaF>^x{|Ut}m-35W297 zCR;k^IXVA_o$0x~6fe3nie;l@;d#Ov%09C{L}Q>GQVR>3522Nf65V5{f%JZCgv zAk?Ri;+AXX8lgr)*Dwo1@=J*;>>e!!3{ZOm0W0#nK0Lw}LJ38ARKdgwmcQ&dg((WT zA?-E%9s)!d?)$)StpVtwDMPvWELsU*9)Av3IO|m3(UHu>MreK5g7sP4#jfer8b$>- zPP}@>*y4?Y369OLlJOdyD)Jm!3wjk6Vmup=B4`TmEyiZW6 z*P=##xNG2c8l>OVY_tViHpDA`%xPhd{*O~1#o5yw&jqXY3)fBD4ey&qSmqY1HW87y zD|qOcJZ{)rwNI&LKm$tD33#}H+MIj#brK$3?fCc?N~ z^xy>rRG?9BAG)2!wlY=;d$}ffffduxXPm=;gy;{q97F~VYoW^nv zCjor3&QbtyahOuEcJE|EdLz+~7<+$Q@Jzeqgar#zA%XnGV~AM$L6isHaAzO6 zw7uAikOPTq=LNfBuyg|{Y0d=^*nS?5>xLku-kFlMebEnQzFedK3dNz7!ZF*??EG&N zY}Mx$8?1=#I+d6%vI|ahH>UUVz(&;=${+1SCe`-ka;KibXNWA*yk?6R&_aCWQh4wJ zMw?n?*39tD^|*H|2j%6Lmz%6j9H^vtpMqJyzl14}Wn*14?F%Ka$F)g1d!bO`yPHAG zTMMzSWH|8U<{-LflIby?{EH?zYis@H%Uc}~yzF=NU-J)cn1ogt$GJ`}QEa~(0IV|~$&axY_INGY# z`${YUbMAh9XvhWfN!Vb;$G?DSym^k^%{+5UnAtLp*xLv(aeo7YOloyl4YJ6roBSOU z)lv%f_cQB%O(CrK*pk>^?YEH5`Z*QW7!5xAy9qUXRk)@{gG9e8Ug=+eA$d(`kY1x! zUByQK=sq9PwnduNrlL~KD6CRo4as8_CSTB@OSW+ir%FT?4@GX1Fq}P-T?DnV5vwKR zjvS1HQ~jN1-T1V%iUJO&UK^Ep>eS%eKWqM`pn3!wtuJCN%V>^zW%f(zSp)*3nf`01 zv1g(qX7Q7!5f*ToeoS+Q^}b%hil^Q7?FIl*K)_Yijf8vakfLT;*QoTC#~a9oSK)%k zOJ@hu#x(F&8*0yf3)-Cs<~mqQaepT^*v>U}SMY;b-wPR@6m^`$x?QnImw7x5f}HDSSecm0z&;+i;pN2hTjoxw^GM?#1lU}O60T;Ap3{5rj`)71L8D^I-X zk!j?j6as`Hx6H*7Q(g!siOWi(B^OMd;dzQl^+gRgH5sM2xB*X|TFT0$lo3ea?H2{~ zcb)#X(NX~4kzKHYb#5MF+M`?Xz21#CJ>+DJ-NK-l%#1PmEBEG3UY>@y$t1xIGt$4>D;85&37DONA0G$D>X5nd#$d zH3H*of=S=CIl|3!%+A6!c*mZ9`kMNZ2TVe~aQYU*4DM$u8~DX&&$g?Q#$Vp*bTuv0 zq}2qL#o8+X!)`DhcVh{{RAp>o)MmO4E%;!KVy2E|7q#ux(#ZF_PH1PQs4J(a%gp+< zM^D{12V`ndpI0x_{7r?eu#%VH?HSGRdly8&?`pcK1(;jQ9a-$Yny>lt9`zxMJJU-J zdpYz~-k2OYlHty_Fuis2U3NVMv%JqGUMoZ4C@7kX`K$r1pk3grwKxohxG+t^f{ zd&@pt2xQ58kAy(a$>!s10=Lys^C2Zx7ll~I=EEe_Y3X?Rh~+y^bY=_q4;Z=r#ur}E zd8C`WKuK z=WwaT%QOqLJQ#DWgJ_6Ky4QFP6xLXIz$e=#+nr*r?v$p%(zc#?d)H5ily?BJ!aZmPH~O|Fj4zTa&T8>b3`=<&Fi)?;88N zFFVBqVK1q*Ct@tB7M-8mI+`4>NJHFV*pjS$d-+u*Rrv*ixYyP{J=$8tFz5luLZd%6cCDsjU zy}oY$MXV#FBm`sZQ4@h-;7-$}JdZNx?vVFQvVH3HRyqKR@og(ma94oGM$)62D?VXF zQPSh8qVGeTUe^K|&iJN7qSDeK8M&Yog=IAJl9zmk{mzFVKIs zWB!Ae4^8{xuJ8}1=J@k}IKloe_}1Rh!q)je(Oa7&1Gz1F_|U#Hwd_SW;`!q0#pW6; zJVtU&=2Tf33?U;yTSp0+5?NEOV919(H&-EwKxIj<7q7V4z!a$v}k7o zVoit80Q=3htchtq%;RvwocZe=aOp8s6?Z_!#T>6_?taDRVXN;LIJ86wqt<|R- zm2E;Soe}b6lbkW*4YC|5D3u|Jf2=x+RnxHaVcsX!6SY;VgEWDtNEUNzWAey(6Pw%-<`5%i2_$O|{;@D%040!ih5PDBBe4g-jmgG>FegI!WIalI zJz)l(b`Bz~J(I;6$%dkBl|;l$8wmkn&!3gY#Y#6TDXX7vL4rhd_v-RT4K^9>g)M-O z)_Dv7V)LH8oaapcPRY?EiMG8tAm z5ib-o5p8~WKOT9uU}_o*E07wu5PZHxcv9V z`rG@%+Ys3K!4d%3_HzWlkdQQ8A>)Q=0trX_i7ld`M^PmB;8CFqStS@K4yuyr3#oS0 z8P^9AydngE(Lj;W{OA#r7zxwmir!Q|SnpZ=8}XqyYlcw|znZoAxMdq3izQ%N3`D-hrpm&yKxIMW zb@R1`B#yfz+YC`Qo(P-=M))rT$uBBkYS1fjCmPj-4ViUSn7QqJ=2q1ZhRwjgE8@2= zoDixikq4YrGjf(aTZqQ6VzTE?v^@)`e9S6?qjTsH11@8DFgz7lyj6}3SBiKbukav6 zLv0!eMRQT0qe?VlZ<<17ezKZfoSb@w`)Y+=u6}~$O5AeG5}XY}zyBYTASFm@a#U`H zjw6r&0HOZ^_ut0O(Zs;m#PL7>B`X?Qc35L5zGrpmig4Pa$RwQ-^OYumD-8+-0odY> zLH-1&Qn&G&k=r}h(~^Zt0I-N+jYb=y2h0&R_^wK9wN_HJhQ*B}V}7Di5^_IKJZ7dZ zqKa$GB9mZ=O-`rTk2(M13y?y)R<*-ygi+9ptqki4Nq{KeB{bq#W5A3I+v7G zRHB#I2EAmh@Gdjz!+YgkEk3G6!x$1AEv+Eh<6())B-DQ-QzdI7Ds#9+`V5{IB2udm zhR33W@0X9;gq@X59g7G9(P&AT^^0PE&dzIrg=5H-{T2-b6_pSQvUp*b)0wT#YQE>H zm1?j?p;7-<4PTV*k9mwUGFagNTn>}(X0R;2h!6x7^1w9J!NtnhSYI#C9NZ`}W)<5^ zOn~L#bhvymaj`U%U$5>sYpXCCF1Ghb1DC(bIczVG*Qo$ZqcsZTz-`FP1L?Fa;Dg2; z71RM0%mLG@?~dG_j(U`<^5`+CUf}A_Apvm8Xq||YM~bwh@ptRtBH{w*-rdXa)Xi*e zCcbT~taOyjj8s-GsY&U0d1Hc&)Rc5qcJLe;BhNeR%h&L43{ZhSCA<)t4%{;s2@P|R zJx)d`a|!=Y0whD)YE7sLn6~Q@TV^tIj>qK&Kv?^-l@@f)1V0Z_dvkT`2n=oRQvjtY zBmVpYyMy{$oJ(OJZoU2H!T8J8K2s3F77ShoAvhDvoJG~I@C>tNuL&tx!n4kIVL zQ-<_tF-&2Q-<1<5_!7*LeaEaROxh*sUAyvQJFi_cBs})^RGbMtdvz$dNR$Ta5xOHn z!VRj;(mi)kk@y1WOD8^yEu=c=efMUkG z1UiFcBS(f1*$8t5EhNoj1%E{IAPvt9q4gOH_5{9W@M)MhH2Ng^t~sq*M@NN&)WEUC zRn+EaX-?LW>7j$V3G>Z+%&7KuO!Ld(x!8D{6=pq@AQz$F+rLePwaACu-N zMt1GVvCB{1K==BjCBW(X+#WT~i1=+WBH){^^nQLow>4hN`^s$;YcyNW981G-wb(ZG zu{v~l44UnA4eZa*x9*-pv__f0dYpA@ID4ljh|W+ro3=rL1(PN9E#RpR(*7^>?!Q6g z{MJP{s!74A5GaTpf2d6Is0=OU5v9Ne;yzJMP>Mz1+8bVjO$$17Omyxm__|An(ZRHUfs>yh>(-yLW(TkJf{k1z&jEvJ51LX3Q(4)FFLX>uO#li@HyfFd*r$LEKx z7XFQ2j{zS8z#lt1htJ{S95xE*x2-y**TfAeM?h2%9~YBy&hV&DB8<(YnLeEL7=5&m zqbr-Xi&&RRebO~S9=-?8tnhESAdHhgfw(oj(tf&d4uy*6-vj2730cVJN3#zKVyG2H zF)+xo;K4%pvJaZMutW*1O?D1E7InO@A!YQL!Vex2QIs59{uUgDL;@_Ppt4lQX&1Cj zeAzLJ{650NnS#X|W3BV=xAes~I$;(F=nY-v2G^ft=KEukxF82zxegklTpW^^Nh8{_Cpf2 zy82NRZ+;RKa+ z$0auj=+G=C3zSMyF&=L%SZpw@aDo2Z{(19#JGSPk>)r7^Bl$`v*R|){+Ty{azO}94 z+1TQ1VD&w4dsfxu2a=?-(d*FH^>w%ZK(shh> z%cNN5{upn^46kRqJH%R#+GbDlu6AeL^TWd40_3TGlGZ^2ew;1B!iU7&!QtxtM*l*? z{syporN>9&5b1$K`=5F*DSaX?FLnqZHzw^s<4zYAd{|(|y4c|3{tyX3wnVX=oEHZ6 z#^PSQ5=&pg?$18NZtNh5>j|z8c$J8YJyYVggRUxvHm+UH%cWwVfw1c_t4lPhOY@6! z^XgO4@@3HF6T{{LgXY37GNvjN#=LsJGT2~5`C$Jw_O!^+?`|s$qu_SxcvqvL&uYV^ zG89uW|8`?#mIb>5=jh~Xrv+|Y;`gpS4}C@|)+ii*`MZq82sk8yQitznMar~!7fm+= z2#LTiLn7Az`*q!ncQ2n7wNjd!)&x{DWT^oQ-Y%O-z|B>iDHvh$*aCAEvd-S<(qMeD z4f)?S|C`hK1WC=x88Tbo7gMkN!%3+|;z>tVcQQ1zvG#KqVrB~~KE!TicJV=h6E}Wo z?8@cPJfj@w
0Gj$|{oJ;?TSs&ygB?#nGSWAI`!QDV6d-ta z;WlfJ;rOR)=EHA%#%9g<_ z%MD6{I668S1uRz9e7#|*2k!jk^&v=ylT>4{6OJcD0`;aB{c`7Bp&Ady=&;b;u`u4? z){3=a&%<5WDvak2Ok72K_4t4)S`-uE@TDk|NMPeR0*7rYQh+h3medh1?RV)rXnhTUAf)LEp5kCC1cxR-A2+~fSwVD>RxhQR}#7{<@ zJrt|J>d$p}F}cQulSn2&I9Sw4FV+auFfx>$6^g;36twk9$|)U{;pWaM$7%?Jqm`6$ zNG215X;lvl65T}PafT!acGe`8;N=>50~neVG`rEMg-$8$&G3WeH*F$ENUtN*2PL`* zP>c2x8UR26>;GlC;lG}NjcPRPqP0~} z_+DpzH#Tnb7H!j1-7^u()LXwQK-Oq9$(!MIE~y$jopZV~+}vJoX1g?vl zWV0xRGsP<-@C*MGlM@t!8$$as97K8VN98f^(7b^lA@Nn7Wq2HQ#GNi|fJ^zs= zYqyo}NesRq2Dco`&kmHtEL~cU9g-l<1|%Cup=Ot;eF~UHz#tou!;~F0@I+!~XM`ei z_y)X83)&7E`f%@ue?4~Zr+>zqc|C&I_B%3Tq^rlAUof3=SzDMPMm6!$_^342XTiYz zk>&id#YO;NZa}Wr?%#m?G8QHzauKnM2D z`%LoZo-a=IzK_rMHv(rbqGdYZicaIMwr4tTX;@+phjfVzfj)XUUvq=@h zw6`}+TniVX0+dNzG{ju)(v}+_^oKLN(AkyA=8QP+l4gd1;r@I`&|B3r|oLI79VrI?q}68@m%R|o;h;eaGB=!th_wy@hlQhfZXWDV%dl+ZuYfR za~+^4-!`RgGy>CD+%x&CtnWpF5g+Nv!rOYb-fKtK#Or1Si^I7d+yX2q!N-5QbMmQ@ z_q?t8#VSe}=M}+((l&@gwvCCj@AFtSQAy^pX}wanF2IYC5ru~=AIu#JYY3x1Lb&kw z^5uUqAxa~ceIU3KE7EYAn!_UkhQXlf+T(c?I%@L?`T262;N;|G*?N?0GDEBOvJBi- z{qnZxTCA6zc9QutW;rzgORPXXGqq?q@RA*Pv)+>5-$ip@AZC$MWx>Kg4LU5_AY;wo z`1bjKRdycmRDb^;zbGT3LiU!u30c`Pva_--uDx9&Wn9V3$jBzj$f)R&?6R|WArcLR z2t|YcxqW}CdrAF|$LDeJczm9Z*E#3?KJRnxIiL5(!Gm`+k(pnO8M&-)GZSYmh}SRA zvWaWl@hFYOY_^%=Y(;XG4M)+3twsI37=C4Mer3@uUYzS1Rjf!o%h{$*XIiV<3Bil$ru}zx>3J&8Aq^WQ-3u;uDJVeg$1+A8!Ip9;qCf%;49PA_i-=%Q(p5jp-6uINlxs~kEIa5YR7R@uOrNb* z!ytz}_u$}z%f%L{>y*{`tRgCF!f7KBBL!vG>l5bvEB!4!hXuk2nVlBW%{(u|Tm{ms zj9+)+@*Fj$y>PG$`?7v~H*bXOP}#>}magN`kue*twO9E<^joiitNMp9{{KD_|`eHO|w=@lIcBC(L(zPH41UC*PBD|Jb~6lK@rgnWdXKR)$bR z8qR%h_TElHn00Pou+?`StrJPRX`BxI&#ycD~i;8o7e8fPC*9O%B1Lt-P!us>FdqGp3?pr=bI}IyR#lIJe8UMd@mL zB(Ct`oERI`xq;Qe85-4g3YRV=;oO0FqRfZ-lY8ko50!Udrhi{hx5Lg2q*=@)~sxX51`e>dnQqC+uNzQNkH! z`z&(H#Z4gHJKOGX;K$PZ)!@EiU4lh^!oq`4C8v!;v?h_wYS;z|_Yy*qMM`ZfMb%}9 zuidVsO3xTDI%V*cP;iRqG=q!l?CbHkpv@`nKC>D>+r`e5(QJR0#fubCts@MXWzQNG zO=W@-q#j?Sm@=(*r>9}4^YB@(G~_pv^4*-gTPa~$vtcD-|7vqYxUWp+`{Q@F^~5IE z9?>k0w40j*EOe4gi3e=18yWue0@Iq`2Td=Kwy{9K!?PSMD4n>fMv z;=B3JIj5sn`Rcb8-d(xxwAda17f$V&k@SH&@ymE^&J#M3Y+Eg+g^vY(sDb)U>Stowh$qBDf9wl>^B^#x)dbOQc6eYfFd0&Z3d~NBTdtujZS=6Y1 zfXL{XaS=3k-rx|R9DqK{ZH}Rn7;fM?RN+N?eyAk>bTlV>G!@}NSitzu}XO4U~;UIA6Zoy_nn;S^{_?ykBInk_EY|khjuEfMu|77p; zaG|v)tzUR9^7=fRQ6{^{|EZls)z?)HXY3vgzOC%+O9WmOz?{rl{Xv{l&(`R9slw~VT7AjzARBI`&)t1 z((o5*@@{Hs3pttR15hD{*1V)j`H^Ny+=nf+6?Yl-_mIPAxB^v9)IyC3V)fKP+T*hA7&M^}e{0c2ZE@ zEg9y7@erM$!RoC7-O_eLcik^N%Yz0-ZZNWKPAJ5yyt#3-<#e}LTS!7+Xo2zN>9!6X z?JJ2miY{)ciRT;e@-*o>+EuCHDQbqI2hKgH65lrMHzd1s-hc5?np{J=XrGU#~cXz%zx)*RVBCYQA&2$;LjAdD8dz33%*k{=MGRTAB`d zq+h}PjYdC>S!HukbxqATgtSSK+f`Qt1uNG?Up~;ht0zE1xN5tuNES!C{ASpXT?jh# zx?9*nneIZrhx#Lp#3yF6Ox>O@H54YTq~jeQJmg6EtXC_W`_;9EdTN`I<+W=fPDa}E zg5ohpeB*-02kLlEGGQB#$g#J$jgZRGG7~ZiaFNNELfb6^j^{~IRG4HX%#Kpt!atVZ z98k@am1=p2RXUtg^{zRTy~&`Dqf68PsUEE)WGI1k(6`smZkGoQf6p4TWRNZCaM6G}UK zKFo9CfzcC4VEwBo`u(& zwXLNkt8z=Jc-WI}pVf1D&!ra`=tI@QO*=SH@nB>Md%u@tqaXMIy5M4Z_pYmGDcctW-4DI7PeJ=Fz4dzR<$}M z#rMvXnau-y{x@~p1m+pypyDCTGcSrx&y(P-d1Kcv&XxDAj9ngjkBr87C-QWKgmv?tF{7J*VRHRd z{+R$)=RA(qt6|Kh8i|^&$h_K#YF0ZLiK{1lSNTqOQl+9yd>QNIu_?si46Z7)#9Vg5c9_L@0wS0G8lh>N}sVh$ecIF%U7lJXB zL1%h1daJ-1%!`1+7PLdX1_#>$*R!w%mCit_TZl8_GY?xO4#9*v3D?-BID(Yszbxx0 z6!$T(7C)a{c=uS-F5j1h$xteW)5NJH=UKFLL)@sH-?HV63^IQ*PHAX3-r?_yiX~)j z$)lV)2>JX}h2_<|)4a-k`7iG1B&B$MI91a|#Vq?FJg=pxis7xjiIx+6r;~|yGXMA* z)fz>U#oCgWyVhc@?(|zz=oQxcgCuovzGI>tMG%T7llO!zqKPeD%^IezJS0B7V*MPu zlDVIp>HLKd!B1wcS9Km351rj!_}u9Ka+`-Cz7D$m+dds?k99#Y^6^FBXlZy52=#uB zW}Sb|%#Boqtp0v&mG$hAGt}Bep**Z)tHIwPQCyY9PjKqAl}6@z@XJJNt%Wnb4|elZ zmtAnso*7#!T(;Z5a{pjmYnTl4bLz$(sRN2mEM)2^kk<$FGq0ODQ5h;`ed|d-ahLLuY5pygOIJ zLJ~bF9!OSw6>q3^vxt3J#6kT0t$xgFm>FwgYVeSf_-Fj-)#wd_!wu&{KO9ID2`0hr zt+AwE=53D{q%XypYDn{48W<$CeeE|{SlrSqRpR~f?OS+fr~{3`6C)QhcsghHD$kQr z&(NX>&(=ist_$Q34P#Ao)lK^+)z!^JzI_$1i|tN5UhP)olWRuVsJmi(Zm@jWGsLP7 zPHt;tr6FITBAz)apxa!KOH>;*yDE*0b9&=G?-! z1HCWuQYMQ8xYK;%OQzHIDCfp5znns2bzP+fpIeVlBS|>nNs(gvH9a4S`#D`4RIgxp z6$6t=n{Piq=+3-#{eu;l}zb-Nt`?h3;FW0X<~8x3V+SUV&qec_a3up ztwHPErpU0BP67L4Yu$z_;-g%}@WtF<>0v~2AN`l0u+0Xn@Dp^i>bEJM;$Sa-z@6Xj zVZ22yW_kK8cD;5>R9{k@SFEFso0?bKGybFbnp9M#qhU7n{a9YmQ}!&rwK^HX9CBtn z{X|A>73_BwLf%uDHF-1WwDRSAkcpb6uX@)2f$CRgJvCTL?6m2RxF)iy^)f}wp7GV0 zk5h&-T^Yy_PNolCmt*dHaAxHl2dT9``INlwa+bL@h?Ly;YBe!2zt-C1lq^kOWk>t+ zG`=AXdDyXW;%9LN{F`DGvC}RdkzWU($Gu~ZRvPPnV|I9PcEfwzT{alE){vAWzB{98+Y&Go-MG=++d`md@gpMql%2Y}nM?d7s?m>=BT)6E*@ z<@IZS9hRWuHZDMRWqOr6_CN3v5XBF(*9iP8KZNz2W=8ET6FITPdL1o{vaU` z=@^?dH{Sjx@bkE}#_6-y-pdy|Mg~-+1+EA`qHI!oV8>E2KEv18Znd?^9T=5o|MsKu z@IwmO3jG^;2dS#l#VVQK^m)TVENn`L4@JAjO7c^h zr6_+US#Tevi+xFyPbZ=7%*3nK`D{K(?>!^=5W|G>p=473`FBh`BcX!JAn6kxyAiL^ zw5R^mJH|=Kg36+4ZD!Ax29j0?jiX*qdUStp_Uwp+>X;rbebvG_h0(|TIhxdb>+WIp zY3vku1P%(AANwDY^n6iiKRJD<+DslwmLD;6 zv_Z)sQ}gNUiQWdc&s64l&-v6U{IaZSn;kQxT8-<@ijOm@nXj;DX0tK6k{9OcIZ2&X zP@8U>c^iK$0K&0;u%jxeOZT;DO0k5_QLgWr${wfeJ8ARR!yTyZTBaz|=aagoZr&{t zz(2(G5IG&dOK0qQaz*+)?iP_vBzVrBa;5n;|F+wbsK5gRHMYy4AiWE=fhOG)8PO>W zVWxIuLPI+4q;87VoEsRgv7NXfnuY^xqZ3>hud)5_$jqL$VGU9;sNo16(fm|s4}GNa z)`MS^f|FkHmOKf)zqVEg&qPl@uAbOJ_e-v+wF{TXEZQjC*-HB<1OzM^h%6fT)7A=8 z8v=r)MIBAk-$xM=;+6~->XK+6Yq&|5MEr+4W=!B>L?3ikk~mD2vuT&tg6&ue{az_> zXQ&sR=X?oK6SsNn$IbW$9 z)t;)E&w5~MO*c4M`zBDFu9sruWp|U0le|k=xc1lj`OXU>2XL<3ae@Py*ef<&NxRZ>ymRn=710=ND53+CX3+Kk~l(BFKVC@lS#c%pK-x19P>$VBzfG>g0uq zQZO>q`W;2S!vUX2MBwizS2sAXzk@rk7t|Jpspi>Q$jo7I3rOw)IgW-v{*HjVxm!5H zd|=L)GEf5-&14_3D!A8l7h@eTe#?ntVd3EF0JpFJ31TpH_`bpI9{~3=fo7pV)8R?* zCx-R!5R{Na_j(u{94fmG&Nl~4 z8}t}4Uk8oxj0L=5 z2~ANqnBo1V=ywR3lq_(7%Uve?C_SjqclduPi*`Ur#AjACiVNJ?0z9S*_jGf1{;?$c zy|q!EL`UYBCv^e>={3SMWG%)0Nob-_9f!DS$>j{HW&&_vQ|)I#BHA3#P;jW16EGq# zIMj8|Ho6>O;X4Pm=H_`YYv<&(FKCw}lm=i*KMH#PM+T#jtf1CTR&K7a-*Y`-Hq{Z& zxci{JA&KB(1lLYMB07Tq3fm=ZG{xr#2bkBZ15-k8Eq}z`G%J`V9K)he&3;vSFqQ!L ziWqQH)FNT0AQ2N0e*n4p?!g(+?J1T8g3f^@Jt_^>PC+7QWA^6!NYjo%LfI54^BYHR zpymx2TTw}|b_x=4J!yZ^u2w?by3>KctbP_GLOE-1)PIq5pFyf`Op(POfYQJPJo#rq zBBK8lxl3mDPzsqeuvh}Y|FYWeK#>U5?EPfIJ)y2%wr-v-d-~=enMH*@coSO-69mG! zg9M*QM0fGOQ+J6)k#^dY5%tIa3KC)W5Dn|*4tH>I@P~Qs=M@Ja`3c^D0DAVvS<+v$ z!hfLc5~s*mydn#NFg*zC=vH{X<`3eaHc*#6tdMep=lW4#&-cMdk8Xt{P5YB}i9*$B zr^V1wOdV*b|IQ5T*Ie@R)o(ok?i#qzqnr8PQ2!5EM4fxdvUc_o*<%nvx$LJ@`?o-7wA+}d8`PlL%^fic@FKtsh?;0Heo z5^=I`uS%lR(8QreOKZhbBm@BG3vlT1*LY-4obz6`mEI!TxCSUhfC7fgp9P6X9se_B zmza-73zFA?iUt=<6K#I6C&ttIPaWt9-9Wqmaim8Y)2>!G|AIo3r3N}MrNAXU64Z_Y zyr%=*MoEri{XR$epSimLS%2!j4(e2&c;L^Zz@O10@G#BaSZKmforh}Y9X|x~r=JCh z$YtIO_SaaT%0e;f3ot3*V*QCjjj(h72HYhPk1~|=IEZ}+&=(wN#4MISOSD3{0GPx6 zsdvi*nq7kdH4XF#dL-E7{&(swu~=UcCCR`L*$!+N9or=G57?MFb09I#Q zXl+0v_5Z-`lB@CU>d`{LeGBBWqj9Gd{>*i8cCpxL-#x-a1)*OkEoj z&|OkLtB>-xfkj6#STCSYGI;g(p?Z2ie|;&0Isfd>Bz&_1nDYdf)fY4RSLQCE!x6{n z>cF&7I}H=+V)|$3e~$l0r&mbq*!T+ksT+pbrmnvK4BaIa)#E$fkM68@GtsC&))($j zPpFN9oy&f~nCkYL)CE9I1=k4_{(l9DSp7%hE|K-ZL=yKwCz$|~LLcw$X6+*qgk}ue zOgwC;^(Kz&`ZoZ}1U-%(`*L#kf&B=97&ugr7o~4Cj5q*+JODW6ow|ZgBw{@OA8@;>Wk6NrN7bddFhG8WRiq_c;puXDj{)SNkf(d^y=K=o-mCLw89g9W2(b^ zR`nOHUg7UFG;w=9KKToGL1_;h=0S2yb(km7f8heO{{e?44>iDG9?$s2t2Wz%w{utn zQx@jt^Dhh&bRWzvIhZ?Qzc|A-dvNyL1HsgWxs&pXxeU^t|Mo5Fe)}rg>IcEX1WbLv O&pEIP6Sqfs>Hh(8P~^n` literal 0 HcmV?d00001 diff --git a/paddle_palm.egg-info/PKG-INFO b/paddle_palm.egg-info/PKG-INFO new file mode 100644 index 0000000..90c7ea1 --- /dev/null +++ b/paddle_palm.egg-info/PKG-INFO @@ -0,0 +1,105 @@ +Metadata-Version: 1.1 +Name: paddle-palm +Version: 1.2 +Summary: A Multi-task Learning Lib for PaddlePaddle Users. +Home-page: https://github.com/PaddlePadd +Author: PaddlePaddle +Author-email: zhangyiming04@baidu.com +License: Apache 2.0 +Description-Content-Type: text/markdown +Description: + # 多任务学习框架PaddlePALM + + # 安装 + pip install paddlepalm + + # 使用 + + ### 1. 创建任务实例 + + 使用yaml格式描述任务实例，每个任务实例中的必选字段包括 + + - train_file: 训练集文件路径 + - reader: 数据集载入与处理工具名，框架预置reader列表见[这里](https://www.baidu.com/) + - backbone: 骨架模型名，框架预置reader列表见[这里](https://www.baidu.com/) + - paradigm: 任务范式(类型)名，框架预置paradigm列表见[这里](https://www.baidu.com/) + + ### 2. 完成训练配置 + + 使用yaml格式完成配置多任务学习中的相关参数，如指定任务实例及其相关的主辅关系、参数复用关系、采样权重等 + + ### 3. 开始训练 + + ```python + + import paddlepalm as palm + + if __name__ == '__main__': + controller = palm.Controller('config.yaml', task_dir='task_instance') + controller.load_pretrain('pretrain_model/ernie/params') + controller.train() + ``` + + ### 4. 预测 + + 用户可在训练结束后直接调用pred接口对某个目标任务进行预测 + + 示例： + ```python + import paddlepalm as palm + + if __name__ == '__main__': + controller = palm.Controller(config_path='config.yaml', task_dir='task_instance') + controller.load_pretrain('pretrain_model/ernie/params') + controller.train() + controller.pred('mrqa') + ``` + + 也可新建controller直接预测 + + ```python + import paddlepalm as palm + + if __name__ == '__main__': + controller = palm.Controller(config_path='config.yaml', task_dir='task_instance') + controller.pred('mrqa', infermodel_path='output_model/firstrun2/infer_model') + ``` + + + # 运行机制 + + ### 多任务学习机制 + pass + + ### 训练终止机制 + + - 默认的设置： + - **所有target任务达到目标训练步数后多任务学习停止** + - 未设置成target任务的任务（即辅助任务）不会影响训练终止与否，只是担任”陪训“的角色 + - 注：默认所有的任务都是target任务，用户可以通过`target_tag`来标记目标/辅助任务 + - 每个目标任务的目标训练步数由num_epochs和mix_ratio计算得到 + + ### 保存机制 + + - 默认的设置： + - 训练过程中，保存下来的模型分为checkpoint (ckpt)和inference model (infermodel)两种： + - ckpt保存的是包含所有任务的总计算图（即整个多任务学习计算图），用于训练中断恢复 + - infermodel保存的是某个目标任务的推理计算图和推理依赖的相关配置 + - 对于每个target任务，训练到预期的步数后自动保存inference model，之后不再保存。（注：保存inference model不影响ckpt的保存） + - 用户可改配置 + - 使用`save_ckpt_every_steps`来控制保存ckpt的频率，默认不保存 + - 每个task instance均可使用`save_infermodel_every_steps`来控制该task保存infermodel的频率，默认为-1，即只在达到目标训练步数时保存一下 + + + + +Keywords: paddlepaddle,paddle,multi-task-learning +Platform: any +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 diff --git a/paddle_palm.egg-info/SOURCES.txt b/paddle_palm.egg-info/SOURCES.txt new file mode 100644 index 0000000..1f1a25f --- /dev/null +++ b/paddle_palm.egg-info/SOURCES.txt @@ -0,0 +1,47 @@ +README.md +setup.cfg +setup.py +./paddlepalm/__init__.py +./paddlepalm/default_settings.py +./paddlepalm/interface.py +./paddlepalm/mtl_controller.py +./paddlepalm/task_instance.py +./paddlepalm/backbone/__init__.py +./paddlepalm/backbone/bert.py +./paddlepalm/backbone/bow.py +./paddlepalm/backbone/ernie.py +./paddlepalm/backbone/utils/__init__.py +./paddlepalm/backbone/utils/transformer.py +./paddlepalm/optimizer/__init__.py +./paddlepalm/optimizer/adam.py +./paddlepalm/reader/__init__.py +./paddlepalm/reader/cls4bert.py +./paddlepalm/reader/match4ernie.py +./paddlepalm/reader/mlm.py +./paddlepalm/reader/mrc4bert.py +./paddlepalm/reader/mrc4ernie.py +./paddlepalm/reader/utils/__init__.py +./paddlepalm/reader/utils/batching4bert.py +./paddlepalm/reader/utils/batching4ernie.py +./paddlepalm/reader/utils/mlm_batching.py +./paddlepalm/reader/utils/mrqa_helper.py +./paddlepalm/reader/utils/reader4ernie.py +./paddlepalm/task_paradigm/__init__.py +./paddlepalm/task_paradigm/cls.py +./paddlepalm/task_paradigm/match.py +./paddlepalm/task_paradigm/mlm.py +./paddlepalm/task_paradigm/mrc.py +./paddlepalm/tokenizer/__init__.py +./paddlepalm/tokenizer/bert_tokenizer.py +./paddlepalm/tokenizer/ernie_tokenizer.py +./paddlepalm/utils/__init__.py +./paddlepalm/utils/config_helper.py +./paddlepalm/utils/print_helper.py +./paddlepalm/utils/reader_helper.py +./paddlepalm/utils/saver.py +./paddlepalm/utils/textprocess_helper.py +paddle_palm.egg-info/PKG-INFO +paddle_palm.egg-info/SOURCES.txt +paddle_palm.egg-info/dependency_links.txt +paddle_palm.egg-info/not-zip-safe +paddle_palm.egg-info/top_level.txt \ No newline at end of file diff --git a/paddle_palm.egg-info/dependency_links.txt b/paddle_palm.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/paddle_palm.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/paddle_palm.egg-info/not-zip-safe b/paddle_palm.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/paddle_palm.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/paddle_palm.egg-info/top_level.txt b/paddle_palm.egg-info/top_level.txt new file mode 100644 index 0000000..b136828 --- /dev/null +++ b/paddle_palm.egg-info/top_level.txt @@ -0,0 +1 @@ +paddlepalm diff --git a/paddlepalm/mtl_controller.py b/paddlepalm/mtl_controller.py index 3550024..0b9ab1d 100755 --- a/paddlepalm/mtl_controller.py +++ b/paddlepalm/mtl_controller.py @@ -626,7 +626,8 @@ class Controller(object): rt_outputs = self.exe.run(train_program, fetch_list=fetch_list) rt_outputs = {k:v for k,v in zip(fetch_names, rt_outputs)} rt_task_id = np.squeeze(rt_outputs['__task_id']).tolist() - assert (not isinstance(rt_task_id, list)) or len(set(rt_task_id)) == 1, rt_task_id + # 注意注释掉这一行之后，训练日志实际是错误的 + # assert (not isinstance(rt_task_id, list)) or len(set(rt_task_id)) == 1, rt_task_id rt_task_id = rt_task_id[0] if isinstance(rt_task_id, list) else rt_task_id cur_task = instances[rt_task_id] diff --git a/paddlepalm/utils/reader_helper.py b/paddlepalm/utils/reader_helper.py index 0124e63..e8e396a 100644 --- a/paddlepalm/utils/reader_helper.py +++ b/paddlepalm/utils/reader_helper.py @@ -72,7 +72,8 @@ def create_net_inputs(input_attrs, async=False, iterator_fn=None, dev_count=1, n if async: assert iterator_fn is not None, "iterator_fn is needed for building async input layer." - reader = fluid.io.PyReader(inputs, capacity=dev_count*n_prefetch, iterable=False) + # reader = fluid.io.PyReader(inputs, capacity=dev_count*n_prefetch, iterable=False) + reader = fluid.io.PyReader(inputs, capacity=dev_count, iterable=False) reader.decorate_batch_generator(iterator_fn) reader.start() @@ -153,7 +154,7 @@ def create_joint_iterator_fn(iterators, iterator_prefixes, joint_shape_and_dtype for i in range(dev_count): # results = _zero_batch(joint_shape_and_dtypes, batch_size=batch_size) - # results[0] = task_id_tensor + results[0] = task_id_tensor if id in outbuf: outputs = outbuf[id] del outbuf[id] diff --git a/run_demo1.sh b/run_demo1.sh index d6d30ca..3f3d8ec 100755 --- a/run_demo1.sh +++ b/run_demo1.sh @@ -1,6 +1,4 @@ -export CUDA_VISIBLE_DEVICES=0,1,2,3 -export FLAGS_fraction_of_gpu_memory_to_use=0.1 -export FLAGS_eager_delete_tensor_gb=0 +export CUDA_VISIBLE_DEVICES=0 python demo1.py diff --git a/run_demo2.sh b/run_demo2.sh index ca69529..e0a0852 100755 --- a/run_demo2.sh +++ b/run_demo2.sh @@ -1,6 +1,4 @@ -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -export FLAGS_fraction_of_gpu_memory_to_use=0.1 -export FLAGS_eager_delete_tensor_gb=0 +export CUDA_VISIBLE_DEVICES=0,1,2,3 python demo2.py -- GitLab