From fa20bc49f68e501b01590bdfdf09c07aa8566fef Mon Sep 17 00:00:00 2001 From: wenquan wu Date: Fri, 14 Jun 2019 17:01:13 +0800 Subject: [PATCH] wwqydy patch 1 (#2411) * Update README.md * add ACL2019-DuConv --- PaddleNLP/Research/ACL2019-DuConv/README.md | 48 +- .../generative_paddle/README.md | 46 ++ .../generative_paddle/data/demo.dev | 0 .../generative_paddle/data/demo.test | 0 .../generative_paddle/data/demo.train | 0 .../generative_paddle/data/resource/dev.txt | 0 .../generative_paddle/data/resource/test.txt | 0 .../generative_paddle/data/resource/train.txt | 0 .../generative_paddle/models/best.model | 0 .../generative_paddle/network.py | 531 ++++++++++++++++ .../generative_paddle/output/predict.txt | 0 .../generative_paddle/run_test.sh | 72 +++ .../generative_paddle/run_train.sh | 86 +++ .../generative_paddle/source/__init__.py | 10 + .../source/inputters/__init__.py | 10 + .../source/inputters/corpus.py | 235 +++++++ .../source/models/__init__.py | 10 + .../source/models/knowledge_seq2seq.py | 590 ++++++++++++++++++ .../source/utils/__init__.py | 10 + .../generative_paddle/source/utils/utils.py | 244 ++++++++ .../generative_paddle/tools/__init__.py | 10 + .../tools/build_vocabulary.py | 88 +++ .../tools/conversation_client.py | 55 ++ .../tools/conversation_server.py | 69 ++ .../tools/conversation_strategy.py | 64 ++ ...nvert_conversation_corpus_to_model_text.py | 106 ++++ .../tools/convert_result_for_eval.py | 53 ++ .../tools/convert_session_to_sample.py | 57 ++ .../generative_paddle/tools/eval.py | 179 ++++++ .../tools/topic_materialization.py | 54 ++ .../images/proactive_conversation_case.png | Bin 0 -> 190506 bytes .../ACL2019-DuConv/retrieval_paddle/README.md | 51 ++ .../ACL2019-DuConv/retrieval_paddle/args.py | 138 ++++ .../retrieval_paddle/data/dev.txt | 0 .../retrieval_paddle/data/resource/dev.txt | 0 .../retrieval_paddle/data/resource/test.txt | 0 .../retrieval_paddle/data/resource/train.txt | 0 .../retrieval_paddle/data/test.txt | 0 .../retrieval_paddle/data/train.txt | 0 .../retrieval_paddle/dict/char.dict | 0 .../retrieval_paddle/dict/gene.dict | 0 .../retrieval_paddle/interact.py | 91 +++ .../retrieval_paddle/models/best.model | 0 .../retrieval_paddle/output/predict.txt | 0 .../retrieval_paddle/predict.py | 101 +++ .../retrieval_paddle/run_predict.sh | 109 ++++ .../retrieval_paddle/run_train.sh | 109 ++++ .../retrieval_paddle/source/__init__.py | 20 + .../source/encoders/__init__.py | 20 + .../source/encoders/transformer.py | 321 ++++++++++ .../source/inputters/__init__.py | 20 + .../source/inputters/data_provider.py | 459 ++++++++++++++ .../source/models/__init__.py | 20 + .../source/models/retrieval_model.py | 250 ++++++++ .../retrieval_paddle/source/utils/__init__.py | 20 + .../retrieval_paddle/source/utils/utils.py | 56 ++ .../retrieval_paddle/tools/__init__.py | 20 + .../tools/build_candidate_set_from_corpus.py | 122 ++++ .../retrieval_paddle/tools/build_dict.py | 76 +++ .../tools/construct_candidate.py | 185 ++++++ .../tools/conversation_client.py | 63 ++ .../tools/conversation_server.py | 78 +++ .../tools/conversation_strategy.py | 76 +++ ...nvert_conversation_corpus_to_model_text.py | 215 +++++++ .../tools/convert_session_to_sample.py | 66 ++ .../retrieval_paddle/tools/eval.py | 178 ++++++ .../tools/extract_predict_utterance.py | 70 +++ .../ACL2019-DuConv/retrieval_paddle/train.py | 244 ++++++++ 68 files changed, 5796 insertions(+), 9 deletions(-) create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/README.md create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/demo.dev create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/demo.test create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/demo.train create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/resource/dev.txt create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/resource/test.txt create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/resource/train.txt create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/models/best.model create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/network.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/output/predict.txt create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/run_test.sh create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/run_train.sh create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/__init__.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/inputters/__init__.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/inputters/corpus.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/models/__init__.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/models/knowledge_seq2seq.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/utils/__init__.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/utils/utils.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/__init__.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/build_vocabulary.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_client.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_server.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_strategy.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_conversation_corpus_to_model_text.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_result_for_eval.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_session_to_sample.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/eval.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/topic_materialization.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/images/proactive_conversation_case.png create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/README.md create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/args.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/dev.txt create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/resource/dev.txt create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/resource/test.txt create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/resource/train.txt create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/test.txt create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/train.txt create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/dict/char.dict create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/dict/gene.dict create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/interact.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/models/best.model create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/output/predict.txt create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/predict.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/run_predict.sh create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/run_train.sh create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/__init__.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/encoders/__init__.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/encoders/transformer.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/inputters/__init__.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/inputters/data_provider.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/models/__init__.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/models/retrieval_model.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/utils/__init__.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/utils/utils.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/__init__.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/build_candidate_set_from_corpus.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/build_dict.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/construct_candidate.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_client.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_server.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_strategy.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/convert_conversation_corpus_to_model_text.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/convert_session_to_sample.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/eval.py create mode 100755 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/extract_predict_utterance.py create mode 100644 PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/train.py diff --git a/PaddleNLP/Research/ACL2019-DuConv/README.md b/PaddleNLP/Research/ACL2019-DuConv/README.md index 0c6b63a4..df03be08 100644 --- a/PaddleNLP/Research/ACL2019-DuConv/README.md +++ b/PaddleNLP/Research/ACL2019-DuConv/README.md @@ -1,17 +1,47 @@ -knowledge-driven-dialogue +Proactive Conversation ============================= [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -# about the competition -Human-machine conversation is one of the most important topics in artificial intelligence (AI) and has received much attention across academia and industry in recent years. Currently dialogue system is still in its infancy, which usually converses passively and utters their words more as a matter of response rather than on their own initiatives, which is different from human-human conversation. Therefore, we set up this competition on a new conversation task, named knowledge driven dialogue, where machines converse with humans based on a built knowledge graph. It aims at testing machines’ ability to conduct human-like conversations.
-Please refer to [competition website](http://lic2019.ccf.org.cn/talk) for details of the competition. -# about the task -Given a dialogue goal g and a set of topic-related background knowledge M = f1 ,f2 ,..., fn , a participating system is expected to output an utterance "ut" for the current conversation H = u1, u2, ..., ut-1, which keeps the conversation coherent and informative under the guidance of the given goal. During the dialogue, a participating system is required to proactively lead the conversation from one topic to another. The dialog goal g is given like this: "Start->Topic_A->TOPIC_B", which means the machine should lead the conversation from any start state to topic A and then to topic B. The given background knowledge includes knowledge related to topic A and topic B, and the relations between these two topics.
-Please refer to [task description](https://github.com/baidu/knowledge-driven-dialogue/blob/master/task_description.pdf) for details of the task. -# about the baseline -We provide retrieval-based and generation-based baseline systems. Both systems were implemented by [PaddlePaddle](http://paddlepaddle.org/) (the Baidu deeplearning framework) and [Pytorch](https://pytorch.org/) (the Facebook deeplearning framework). The performance of the two systems is as follows: +# Motivation +Human-machine conversation is one of the most important topics in artificial intelligence (AI) and has received much attention across academia and industry in recent years. Currently dialogue system is still in its infancy, which usually converses passively and utters their words more as a matter of response rather than on their own initiatives, which is different from human-human conversation. We believe that the ability of proactive conversation of machine is the breakthrough of human-like conversation. + +# What we do ? +* We set up a new conversation task, named ___Proactive Converstion___, where machine proactively leads the conversation following a given goal. +* We also created a new conversation dataset named [DuConv](https://ai.baidu.com/broad/subordinate?dataset=duconv) , and made it publicly available to facilitate the development of proactive conversation systems. +* We established retrival-based and generation-based ___baseline systems___ for DuConv, which are available in this repo. +* In addition, we hold ___competitions___ to encourage more researchers to work in this direction. + +# Paper +* [Proactive Human-Machine Conversation with Explicit Conversation Goals](https://arxiv.org/abs/1906.05572), accepted by ACL 2019 + +# Task Description +Given a dialogue goal g and a set of topic-related background knowledge M = f1 ,f2 ,..., fn , the system is expected to output an utterance "ut" for the current conversation H = u1, u2, ..., ut-1, which keeps the conversation coherent and informative under the guidance of the given goal. During the dialogue, the system is required to proactively lead the conversation from one topic to another. The dialog goal g is given like this: "Start->Topic_A->TOPIC_B", which means the machine should lead the conversation from any start state to topic A and then to topic B. The given background knowledge includes knowledge related to topic A and topic B, and the relations between these two topics.
+![image](https://github.com/PaddlePaddle/models/blob/wwqydy-patch-1/PaddleNLP/Research/ACL2019-DuConv/images/proactive_conversation_case.png) +*Figure1.Proactive Conversation Case. Each utterance of "BOT" could be predicted by system, e.g., utterances with black words represent history H,and utterance with green words represent the response ut predicted by system.* + +# DuConv +We collected around 30k conversations containing 270k utterances named DuConv. Each conversation was created by two random selected crowdsourced workers. One worker was provided with dialogue goal and the associated knowledge to play the role of leader who proactively leads the conversation by sequentially change the discussion topics following the given goal, meanwhile keeping the conversation as natural and engaging as possible. Another worker was provided with nothing but conversation history and only has to respond to the leader.
+　 We devide the collected conversations into training, development, test1 and test2 splits. The test1 part with reference response is used for local testing such as the automatic evaluation of our paper. The test2 part without reference response is used for online testing such as the [competition](http://lic2019.ccf.org.cn/talk) we had held and the ___Leader Board___ which is opened forever in https://ai.baidu.com/broad/leaderboard?dataset=duconv. The dataset is available at https://ai.baidu.com/broad/subordinate?dataset=duconv. + +# Baseline Performance +We provide retrieval-based and generation-based baseline systems. Both systems were implemented by [PaddlePaddle](http://paddlepaddle.org/) (the Baidu deeplearning framework). The performance of the two systems is as follows: | baseline system | F1/BLEU1/BLEU2 | DISTINCT1/DISTINCT2 | | ------------- | ------------ | ------------ | | retrieval-based | 31.72/0.291/0.156 | 0.118/0.373 | | generation-based | 32.65/0.300/0.168 | 0.062/0.128 | + +# Competitions +* [Knowledge-driven Dialogue task](http://lic2019.ccf.org.cn/talk) in [2019 Language and Intelligence Challenge](http://lic2019.ccf.org.cn/), has been closed. + * Teams number of registration：1536 + * Teams number of submission result: 178 + * The Top 3 results: + +| Rank | F1/BLEU1/BLEU2 | DISTINCT1/DISTINCT2 | +| ------------- | ------------ | ------------ | +| 1 | 49.22/0.449/0.318 | 0.118/0.299 | +| 2 | 47.76/0.430/0.296 | 0.110/0.275 | +| 3 | 46.40/0.422/0.289 | 0.118/0.303 | + +* [Leader Board](https://ai.baidu.com/broad/leaderboard?dataset=duconv), is opened forever
+ We maintain a leader board which provides the official automatic evaluation. You can submit your result to https://ai.baidu.com/broad/submission?dataset=duconv to get the official result. Please make sure submit the result of test2 part. \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/README.md b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/README.md new file mode 100644 index 00000000..100bc358 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/README.md @@ -0,0 +1,46 @@ +Knowledge-driven Dialogue +============================= +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +This is a paddlepaddle implementation of generative-based model for knowledge-driven dialogue + +## Requirements + +* cuda=9.0 +* cudnn=7.0 +* python=2.7 +* numpy +* paddlepaddle>=1.3.2 + +## Quickstart + +### Step 1: Preprocess the data + +Put the data of [DuConv](https://ai.baidu.com/broad/subordinate?dataset=duconv) under the data folder and rename them train/dev/test.txt: + +``` +./data/resource/train.txt +./data/resource/dev.txt +./data/resource/test.txt +``` + +### Step 2: Train the model + +Train model with the following commands. + +```bash +sh run_train.sh +``` + +### Step 3: Test the Model + +Test model with the following commands. + +```bash +sh run_test.sh +``` + +### Note !!! + +* The script run_train.sh/run_test.sh shows all the processes including data processing and model training/testing. Be sure to read it carefully and follow it. +* The files in ./data and ./model is just empty file to show the structure of the document. \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/demo.dev b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/demo.dev new file mode 100755 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/demo.test b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/demo.test new file mode 100755 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/demo.train b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/demo.train new file mode 100755 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/resource/dev.txt b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/resource/dev.txt new file mode 100755 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/resource/test.txt b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/resource/test.txt new file mode 100755 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/resource/train.txt b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/data/resource/train.txt new file mode 100755 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/models/best.model b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/models/best.model new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/network.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/network.py new file mode 100644 index 00000000..cd3f85c8 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/network.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: network.py +""" + +import argparse + +import numpy as np + +import paddle.fluid as fluid +import paddle.fluid.framework as framework +from paddle.fluid.executor import Executor + +from source.utils.utils import str2bool +from source.utils.utils import id_to_text +from source.utils.utils import init_embedding +from source.utils.utils import build_data_feed +from source.utils.utils import load_id2str_dict +from source.inputters.corpus import KnowledgeCorpus +from source.models.knowledge_seq2seq import knowledge_seq2seq + + +def model_config(): + """ model config """ + parser = argparse.ArgumentParser() + + # Data + data_arg = parser.add_argument_group("Data") + data_arg.add_argument("--data_dir", type=str, default="./data/") + data_arg.add_argument("--data_prefix", type=str, default="demo") + data_arg.add_argument("--save_dir", type=str, default="./models/") + data_arg.add_argument("--vocab_path", type=str, default="./data/vocab.txt") + data_arg.add_argument("--embed_file", type=str, + default="./data/sgns.weibo.300d.txt") + + # Network + net_arg = parser.add_argument_group("Network") + net_arg.add_argument("--embed_size", type=int, default=300) + net_arg.add_argument("--hidden_size", type=int, default=800) + net_arg.add_argument("--bidirectional", type=str2bool, default=True) + net_arg.add_argument("--vocab_size", type=int, default=30004) + net_arg.add_argument("--min_len", type=int, default=1) + net_arg.add_argument("--max_len", type=int, default=500) + net_arg.add_argument("--num_layers", type=int, default=1) + net_arg.add_argument("--attn", type=str, default='dot', + choices=['none', 'mlp', 'dot', 'general']) + + # Training / Testing + train_arg = parser.add_argument_group("Training") + + train_arg.add_argument("--stage", type=int, default="0") + train_arg.add_argument("--run_type", type=str, default="train") + train_arg.add_argument("--init_model", type=str, default="") + train_arg.add_argument("--init_opt_state", type=str, default="") + + train_arg.add_argument("--optimizer", type=str, default="Adam") + train_arg.add_argument("--lr", type=float, default=0.0005) + train_arg.add_argument("--grad_clip", type=float, default=5.0) + train_arg.add_argument("--dropout", type=float, default=0.3) + train_arg.add_argument("--num_epochs", type=int, default=13) + train_arg.add_argument("--pretrain_epoch", type=int, default=5) + train_arg.add_argument("--use_bow", type=str2bool, default=True) + train_arg.add_argument("--use_posterior", type=str2bool, default=False) + + # Geneation + gen_arg = parser.add_argument_group("Generation") + gen_arg.add_argument("--beam_size", type=int, default=10) + gen_arg.add_argument("--max_dec_len", type=int, default=30) + gen_arg.add_argument("--length_average", type=str2bool, default=True) + gen_arg.add_argument("--output", type=str, default="./output/test.result") + gen_arg.add_argument("--model_path", type=str, default="./models/best_model/") + gen_arg.add_argument("--unk_id", type=int, default=1) + gen_arg.add_argument("--bos_id", type=int, default=2) + gen_arg.add_argument("--eos_id", type=int, default=3) + + # MISC + misc_arg = parser.add_argument_group("Misc") + misc_arg.add_argument("--use_gpu", type=str2bool, default=True) + misc_arg.add_argument("--log_steps", type=int, default=300) + misc_arg.add_argument("--valid_steps", type=int, default=1000) + misc_arg.add_argument("--batch_size", type=int, default=1) + + config = parser.parse_args() + + return config + + +def trace_fianl_result(final_score, final_ids, final_index, topk=1, EOS=3): + """ trace fianl result """ + col_size = final_score.shape[1] + row_size = final_score.shape[0] + + found_eos_num = 0 + i = row_size - 1 + + beam_size = col_size + score = final_score[-1] + row_array = [row_size - 1] * beam_size + col_array = [e for e in range(col_size)] + + while i >= 0: + for j in range(col_size - 1, -1, -1): + if final_ids[i, j] == EOS: + repalce_idx = beam_size - (found_eos_num % beam_size) - 1 + score[repalce_idx] = final_score[i, j] + found_eos_num += 1 + + row_array[repalce_idx] = i + col_array[repalce_idx] = j + + i -= 1 + + topk_index = np.argsort(score,)[-topk:] + + trace_result = [] + trace_score = [] + + for index in reversed(topk_index): + start_i = row_array[index] + start_j = col_array[index] + ids = [] + for k in range(start_i, -1, -1): + ids.append(final_ids[k, start_j]) + start_j = final_index[k, start_j] + + ids = ids[::-1] + + trace_result.append(ids) + trace_score.append(score[index]) + + return trace_result, trace_score + + +def load(): + """ load model for predict """ + config = model_config() + config.vocab_size = len(open(config.vocab_path).readlines()) + final_score, final_ids, final_index = knowledge_seq2seq(config) + + final_score.persistable = True + final_ids.persistable = True + final_index.persistable = True + + main_program = fluid.default_main_program() + + if config.use_gpu: + place = fluid.CUDAPlace(0) + else: + place = fluid.CPUPlace() + + exe = Executor(place) + exe.run(framework.default_startup_program()) + + fluid.io.load_params(executor=exe, dirname=config.model_path, main_program=main_program) + + processors = KnowledgeCorpus( + data_dir=config.data_dir, + data_prefix=config.data_prefix, + vocab_path=config.vocab_path, + min_len=config.min_len, + max_len=config.max_len) + + # load dict + id_dict_array = load_id2str_dict(config.vocab_path) + + model_handle = [exe, place, final_score, final_ids, final_index, processors, id_dict_array] + return model_handle + + +def predict(model_handle, text): + """ predict for text by model_handle """ + batch_size = 1 + + [exe, place, final_score, final_ids, final_index, processors, id_dict_array] = model_handle + + data_generator = processors.preprocessing_for_lines([text], batch_size=batch_size) + + results = [] + for batch_id, data in enumerate(data_generator()): + data_feed, sent_num = build_data_feed(data, place, batch_size=batch_size) + out = exe.run(feed=data_feed, + fetch_list=[final_score.name, final_ids.name, final_index.name]) + + batch_score = out[0] + batch_ids = out[1] + batch_pre_index = out[2] + + batch_score_arr = np.split(batch_score, batch_size, axis=1) + batch_ids_arr = np.split(batch_ids, batch_size, axis=1) + batch_pre_index_arr = np.split(batch_pre_index, batch_size, axis=1) + + index = 0 + for (score, ids, pre_index) in zip(batch_score_arr, batch_ids_arr, batch_pre_index_arr): + trace_ids, trace_score = trace_fianl_result(score, ids, pre_index, topk=1, EOS=3) + results.append(id_to_text(trace_ids[0][:-1], id_dict_array)) + + index += 1 + if index >= sent_num: + break + + return results[0] + + +def init_model(config, param_name_list, place): + """ init model """ + stage = config.stage + if stage == 0: + for name in param_name_list: + t = fluid.global_scope().find_var(name).get_tensor() + init_scale = 0.05 + np_t = np.asarray(t) + if str(name) == 'embedding': + np_para = init_embedding(config.embed_file, config.vocab_path, + init_scale, np_t.shape) + else: + np_para = np.random.uniform(-init_scale, init_scale, np_t.shape).astype('float32') + t.set(np_para.astype('float32'), place) + else: + model_init_file = config.init_model + try: + model_init = np.load(model_init_file) + except: + print("load init model failed", model_init_file) + raise Exception("load init model failed") + + print("load init model") + for name in param_name_list: + t = fluid.global_scope().find_var(name).get_tensor() + t.set(model_init[str(name)].astype('float32'), place) + + # load opt state + opt_state_init_file = config.init_opt_state + if opt_state_init_file != "": + print("begin to load opt state") + opt_state_data = np.load(opt_state_init_file) + for k, v in opt_state_data.items(): + t = fluid.global_scope().find_var(str(k)).get_tensor() + t.set(v, place) + print("set opt state finished") + + print("init model parameters finshed") + + +def train_loop(config, + train_generator, valid_generator, + main_program, inference_program, + model_handle, param_name_list, opt_var_name_list): + """ model train loop """ + stage = config.stage + [exe, place, bow_loss, kl_loss, nll_loss, final_loss] = model_handle + + total_step = 0 + start_epoch = 0 if stage == 0 else config.pretrain_epoch + end_epoch = config.pretrain_epoch if stage == 0 else config.num_epochs + print("start end", start_epoch, end_epoch) + + best_score = float('inf') + for epoch_idx in range(start_epoch, end_epoch): + total_bow_loss = 0 + total_kl_loss = 0 + total_nll_loss = 0 + total_final_loss = 0 + sample_num = 0 + + for batch_id, data in enumerate(train_generator()): + data_feed = build_data_feed(data, place, + batch_size=config.batch_size, + is_training=True, + bow_max_len=config.max_len, + pretrain_epoch=epoch_idx < config.pretrain_epoch) + + if data_feed is None: + break + + out = exe.run(main_program, feed=data_feed, + fetch_list=[bow_loss.name, kl_loss.name, nll_loss.name, final_loss.name]) + + total_step += 1 + total_bow_loss += out[0] + total_kl_loss += out[1] + total_nll_loss += out[2] + total_final_loss += out[3] + sample_num += 1 + + if batch_id > 0 and batch_id % config.log_steps == 0: + print("epoch %d step %d | " + "bow loss %0.6f kl loss %0.6f nll loss %0.6f total loss %0.6f" % \ + (epoch_idx, batch_id, + total_bow_loss / sample_num, total_kl_loss / sample_num, \ + total_nll_loss / sample_num, total_final_loss / sample_num)) + + total_bow_loss = 0 + total_kl_loss = 0 + total_nll_loss = 0 + total_final_loss = 0 + sample_num = 0 + + if batch_id > 0 and batch_id % config.valid_steps == 0: + eval_bow_loss, eval_kl_loss, eval_nll_loss, eval_total_loss = \ + vaild_loop(config, valid_generator, inference_program, model_handle) + # save model + if stage != 0: + param_path = config.save_dir + "/" + str(total_step) + fluid.io.save_params(executor=exe, dirname=param_path, + main_program=main_program) + + if eval_nll_loss < best_score: + # save to best + best_model_path = config.save_dir + "/best_model" + print("save to best", eval_nll_loss, best_model_path) + fluid.io.save_params(executor=exe, dirname=best_model_path, + main_program=main_program) + best_score = eval_nll_loss + + eval_bow_loss, eval_kl_loss, eval_nll_loss, eval_total_loss = \ + vaild_loop(config, valid_generator, inference_program, model_handle) + + if stage != 0: + param_path = config.save_dir + "/" + str(total_step) + fluid.io.save_params(executor=exe, dirname=param_path, + main_program=main_program) + if eval_nll_loss < best_score: + best_model_path = config.save_dir + "/best_model" + print("save to best", eval_nll_loss, best_model_path) + fluid.io.save_params(executor=exe, dirname=best_model_path, + main_program=main_program) + best_score = eval_nll_loss + + if stage == 0: + # save last model and opt_stat to npz for next stage init + save_model_file = config.save_dir + "/model_stage_0" + save_opt_state_file = config.save_dir + "/opt_state_stage_0" + + model_stage_0 = {} + for name in param_name_list: + t = np.asarray(fluid.global_scope().find_var(name).get_tensor()) + model_stage_0[name] = t + np.savez(save_model_file, **model_stage_0) + + opt_state_stage_0 = {} + for name in opt_var_name_list: + t_data = np.asarray(fluid.global_scope().find_var(name).get_tensor()) + opt_state_stage_0[name] = t_data + np.savez(save_opt_state_file, **opt_state_stage_0) + + +def vaild_loop(config, valid_generator, inference_program, model_handle): + """ model vaild loop """ + [exe, place, bow_loss, kl_loss, nll_loss, final_loss] = model_handle + valid_num = 0.0 + total_valid_bow_loss = 0.0 + total_valid_kl_loss = 0.0 + total_valid_nll_loss = 0.0 + total_valid_final_loss = 0.0 + for batch_id, data in enumerate(valid_generator()): + data_feed = build_data_feed(data, place, + batch_size=config.batch_size, + is_training=True, + bow_max_len=config.max_len, + pretrain_epoch=False) + + if data_feed is None: + continue + + val_fetch_outs = \ + exe.run(inference_program, + feed=data_feed, + fetch_list=[bow_loss.name, kl_loss.name, nll_loss.name, final_loss.name]) + + total_valid_bow_loss += val_fetch_outs[0] * config.batch_size + total_valid_kl_loss += val_fetch_outs[1] * config.batch_size + total_valid_nll_loss += val_fetch_outs[2] * config.batch_size + total_valid_final_loss += val_fetch_outs[3] * config.batch_size + valid_num += config.batch_size + + print("valid dataset: bow loss %0.6f kl loss %0.6f nll loss %0.6f total loss %0.6f" % \ + (total_valid_bow_loss / valid_num, total_valid_kl_loss / valid_num, \ + total_valid_nll_loss / valid_num, total_valid_final_loss / valid_num)) + + return [total_valid_bow_loss / valid_num, total_valid_kl_loss / valid_num, \ + total_valid_nll_loss / valid_num, total_valid_final_loss / valid_num] + + +def test(config): + """ test """ + batch_size = config.batch_size + config.vocab_size = len(open(config.vocab_path).readlines()) + final_score, final_ids, final_index = knowledge_seq2seq(config) + + final_score.persistable = True + final_ids.persistable = True + final_index.persistable = True + + main_program = fluid.default_main_program() + + if config.use_gpu: + place = fluid.CUDAPlace(0) + else: + place = fluid.CPUPlace() + + exe = Executor(place) + exe.run(framework.default_startup_program()) + + fluid.io.load_params(executor=exe, dirname=config.model_path, + main_program=main_program) + print("laod params finsihed") + + # test data generator + processors = KnowledgeCorpus( + data_dir=config.data_dir, + data_prefix=config.data_prefix, + vocab_path=config.vocab_path, + min_len=config.min_len, + max_len=config.max_len) + test_generator = processors.data_generator( + batch_size=config.batch_size, + phase="test", + shuffle=False) + + # load dict + id_dict_array = load_id2str_dict(config.vocab_path) + + out_file = config.output + fout = open(out_file, 'w') + for batch_id, data in enumerate(test_generator()): + data_feed, sent_num = build_data_feed(data, place, batch_size=batch_size) + + if data_feed is None: + break + + out = exe.run(feed=data_feed, + fetch_list=[final_score.name, final_ids.name, final_index.name]) + + batch_score = out[0] + batch_ids = out[1] + batch_pre_index = out[2] + + batch_score_arr = np.split(batch_score, batch_size, axis=1) + batch_ids_arr = np.split(batch_ids, batch_size, axis=1) + batch_pre_index_arr = np.split(batch_pre_index, batch_size, axis=1) + + index = 0 + for (score, ids, pre_index) in zip(batch_score_arr, batch_ids_arr, batch_pre_index_arr): + trace_ids, trace_score = trace_fianl_result(score, ids, pre_index, topk=1, EOS=3) + fout.write(id_to_text(trace_ids[0][:-1], id_dict_array)) + fout.write('\n') + + index += 1 + if index >= sent_num: + break + + fout.close() + + +def train(config): + """ model training """ + config.vocab_size = len(open(config.vocab_path).readlines()) + bow_loss, kl_loss, nll_loss, final_loss= knowledge_seq2seq(config) + + bow_loss.persistable = True + kl_loss.persistable = True + nll_loss.persistable = True + final_loss.persistable = True + + main_program = fluid.default_main_program() + inference_program = fluid.default_main_program().clone(for_test=True) + + fluid.clip.set_gradient_clip( + clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=config.grad_clip)) + optimizer = fluid.optimizer.Adam(learning_rate=config.lr) + + if config.stage == 0: + print("stage 0") + optimizer.minimize(bow_loss) + else: + print("stage 1") + optimizer.minimize(final_loss) + + fluid.memory_optimize(main_program) + opt_var_name_list = optimizer.get_opti_var_name_list() + + if config.use_gpu: + place = fluid.CUDAPlace(0) + else: + place = fluid.CPUPlace() + + exe = Executor(place) + exe.run(framework.default_startup_program()) + + param_list = main_program.block(0).all_parameters() + param_name_list = [p.name for p in param_list] + + init_model(config, param_name_list, place) + + processors = KnowledgeCorpus( + data_dir=config.data_dir, + data_prefix=config.data_prefix, + vocab_path=config.vocab_path, + min_len=config.min_len, + max_len=config.max_len) + train_generator = processors.data_generator( + batch_size=config.batch_size, + phase="train", + shuffle=True) + valid_generator = processors.data_generator( + batch_size=config.batch_size, + phase="dev", + shuffle=False) + + model_handle = [exe, place, bow_loss, kl_loss, nll_loss, final_loss] + + train_loop(config, + train_generator, valid_generator, + main_program, inference_program, + model_handle, param_name_list, opt_var_name_list) + + +if __name__ == "__main__": + config = model_config() + run_type = config.run_type + + if run_type == "train": + train(config) + elif run_type == "test": + test(config) diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/output/predict.txt b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/output/predict.txt new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/run_test.sh b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/run_test.sh new file mode 100755 index 00000000..9deb95b6 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/run_test.sh @@ -0,0 +1,72 @@ +#!/bin/bash +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ + +# set gpu id to use +export CUDA_VISIBLE_DEVICES=0 + +# generalizes target_a/target_b of goal for all outputs, replaces them with slot mark +TOPIC_GENERALIZATION=1 + +# set python path according to your actual environment +pythonpath='python' + +# the prefix of the file name used by the model, must be consistent with the configuration in network.py +prefix=demo + +# put all data set that used and generated for testing under this folder: datapath +# for more details, please refer to the following data processing instructions +datapath=./data + +# in test stage, you can eval dev.txt or test.txt +# the "dev.txt" and "test.txt" are the original data of DuConv and +# need to be placed in this folder: INPUT_PATH/resource/ +# the following preprocessing will generate the actual data needed for model testing +# after testing, you can run eval.py to get the final eval score if the original data have answer +# DATA_TYPE = "dev" or "test" +datapart=dev + +# ensure that each file is in the correct path +# 1. put the data of DuConv under this folder: datapath/resource/ +# - the data provided consists of three parts: train.txt dev.txt test.txt +# - the train.txt and dev.txt are session data, the test.txt is sample data +# - in test stage, we just use the dev.txt or test.txt +# 2. the sample data extracted from session data is in this folder: datapath/resource/ +# 3. the text file required by the model is in this folder: datapath +# 4. the topic file used to generalize data is in this directory: datapath +corpus_file=${datapath}/resource/${datapart}.txt +sample_file=${datapath}/resource/sample.${datapart}.txt +text_file=${datapath}/${prefix}.test +topic_file=${datapath}/${prefix}.test.topic + +# step 1: if eval dev.txt, firstly have to convert session data to sample data +# if eval test.txt, we can use original test.txt of DuConv directly. +if [ "${datapart}"x = "test"x ]; then + sample_file=${corpus_file} +else + ${pythonpath} ./tools/convert_session_to_sample.py ${corpus_file} ${sample_file} +fi + +# step 2: convert sample data to text data required by the model +${pythonpath} ./tools/convert_conversation_corpus_to_model_text.py ${sample_file} ${text_file} ${topic_file} ${TOPIC_GENERALIZATION} + +# step 3: predict by model +${pythonpath} -u network.py --run_type test \ + --use_gpu True \ + --batch_size 12 \ + --use_posterior False \ + --model_path ./models/best_model \ + --output ./output/test.result + +# step 4: replace slot mark generated during topic generalization with real text +${pythonpath} ./tools/topic_materialization.py ./output/test.result ./output/test.result.final ${topic_file} + +# step 5: if the original file has answers, you can run the following command to get result +# if the original file not has answers, you can upload the ./output/test.result.final +# to the website(https://ai.baidu.com/broad/submission?dataset=duconv) to get the official automatic evaluation +${pythonpath} ./tools/convert_result_for_eval.py ${sample_file} ./output/test.result.final ./output/test.result.eval +${pythonpath} ./tools/eval.py ./output/test.result.eval + diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/run_train.sh b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/run_train.sh new file mode 100755 index 00000000..7925903a --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/run_train.sh @@ -0,0 +1,86 @@ +#!/bin/bash +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ + +# set gpu id to use +export CUDA_VISIBLE_DEVICES=0 + +# generalizes target_a/target_b of goal for all outputs, replaces them with slot mark +TOPIC_GENERALIZATION=1 + +# set python path according to your actual environment +pythonpath='python' + +# the prefix of the file name used by the model, must be consistent with the configuration in network.py +prefix=demo + +# put all data set that used and generated for training under this folder: datapath +# for more details, please refer to the following data processing instructions +datapath=./data + +vocabpath=${datapath}/vocab.txt + +# in train stage, use "train.txt" to train model, and use "dev.txt" to eval model +# the "train.txt" and "dev.txt" are the original data of DuConv and +# need to be placed in this folder: datapath/resource/ +# the following preprocessing will generate the actual data needed for model training +# datatype = "train" or "dev" +datatype=(train dev) + +# data preprocessing +for ((i=0; i<${#datatype[*]}; i++)) +do + # ensure that each file is in the correct path + # 1. put the data of DuConv under this folder: datapath/resource/ + # - the data provided consists of three parts: train.txt dev.txt test.txt + # - the train.txt and dev.txt are session data, the test.txt is sample data + # - in train stage, we just use the train.txt and dev.txt + # 2. the sample data extracted from session data is in this folder: datapath/resource/ + # 3. the text file required by the model is in this folder: datapath + # 4. the topic file used to generalize data is in this directory: datapath + corpus_file=${datapath}/resource/${datatype[$i]}.txt + sample_file=${datapath}/resource/sample.${datatype[$i]}.txt + text_file=${datapath}/${prefix}.${datatype[$i]} + topic_file=${datapath}/${prefix}.${datatype[$i]}.topic + + # step 1: firstly have to convert session data to sample data + ${pythonpath} ./tools/convert_session_to_sample.py ${corpus_file} ${sample_file} + + # step 2: convert sample data to text data required by the model + ${pythonpath} ./tools/convert_conversation_corpus_to_model_text.py ${sample_file} ${text_file} ${topic_file} ${TOPIC_GENERALIZATION} + + # step 3: build vocabulary from the training data + if [ "${datatype[$i]}"x = "train"x ]; then + ${pythonpath} ./tools/build_vocabulary.py ${text_file} ${vocabpath} + fi +done + +# step 4: in train stage, we just use train.txt and dev.txt, so we copy dev.txt to test.txt for model training +cp ${datapath}/${prefix}.dev ${datapath}/${prefix}.test + +# step 5: train model in two stage, you can find the model file in ./models/ after training +# step 5.1: stage 0, you can get model_stage_0.npz and opt_state_stage_0.npz in save_dir after stage 0 +${pythonpath} -u network.py --run_type train \ + --stage 0 \ + --use_gpu True \ + --pretrain_epoch 5 \ + --batch_size 32 \ + --use_posterior True \ + --save_dir ./models \ + --vocab_path ${vocabpath} \ + --embed_file ./data/sgns.weibo.300d.txt + +# step 5.2: stage 1, init the model and opt state using the result of stage 0 and train the model +${pythonpath} -u network.py --run_type train \ + --stage 1 \ + --use_gpu True \ + --init_model ./models/model_stage_0.npz \ + --init_opt_state ./models/opt_state_stage_0.npz \ + --num_epochs 12 \ + --batch_size 24 \ + --use_posterior True \ + --save_dir ./models \ + --vocab_path ${vocabpath} diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/__init__.py new file mode 100644 index 00000000..c97406f8 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: __init__.py +""" \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/inputters/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/inputters/__init__.py new file mode 100644 index 00000000..fc0494cb --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/inputters/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: __init__.py +""" diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/inputters/corpus.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/inputters/corpus.py new file mode 100644 index 00000000..f47338a6 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/inputters/corpus.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: source/inputters/corpus.py +""" + +import re +import os +import random +import numpy as np + + +class KnowledgeCorpus(object): + """ Corpus """ + def __init__(self, + data_dir, + data_prefix, + vocab_path, + min_len, + max_len): + self.data_dir = data_dir + self.data_prefix = data_prefix + self.vocab_path = vocab_path + self.min_len = min_len + self.max_len = max_len + + self.current_train_example = -1 + self.num_examples = {'train': -1, 'dev': -1, 'test': -1} + self.load_voc() + + def filter_pred(ids): + """ + src_filter_pred + """ + return self.min_len <= len(ids) <= max_len + self.filter_pred = lambda ex: filter_pred(ex['src']) and filter_pred(ex['tgt']) + + def load_voc(self): + """ load vocabulary """ + idx = 0 + self.vocab_dict = dict() + with open(self.vocab_path, 'r') as fr: + for line in fr: + line = line.strip() + self.vocab_dict[line] = idx + idx += 1 + + def read_data(self, data_file): + """ read_data """ + data = [] + with open(data_file, "r") as f: + for line in f: + if line.rstrip('\n').split('\t') < 3: + continue + src, tgt, knowledge = line.rstrip('\n').split('\t')[:3] + filter_knowledge = [] + for sent in knowledge.split('\1'): + filter_knowledge.append(' '.join(sent.split()[: self.max_len])) + data.append({'src': src, 'tgt': tgt, 'cue':filter_knowledge}) + return data + + def tokenize(self, tokens): + """ map tokens to ids """ + if isinstance(tokens, str): + tokens = re.sub('\d+', '', tokens).lower() + toks = tokens.split(' ') + toks_ids = [self.vocab_dict.get('')] + \ + [self.vocab_dict.get(tok, self.vocab_dict.get('')) + for tok in toks] + \ + [self.vocab_dict.get('')] + return toks_ids + elif isinstance(tokens, list): + tokens_list = [self.tokenize(t) for t in tokens] + return tokens_list + + def build_examples(self, data): + """ build examples, data: ``List[Dict]`` """ + examples = [] + for raw_data in data: + example = {} + for name, strings in raw_data.items(): + example[name] = self.tokenize(strings) + if not self.filter_pred(example): + continue + examples.append((example['src'], example['tgt'], example['cue'])) + return examples + + def preprocessing_for_lines(self, lines, batch_size): + """ preprocessing for lines """ + raw_data = [] + for line in lines: + src, tgt, knowledge = line.rstrip('\n').split('\t')[:3] + filter_knowledge = [] + for sent in knowledge.split('\1'): + filter_knowledge.append(' '.join(sent.split()[: self.max_len])) + raw_data.append({'src': src, 'tgt': tgt, 'cue': filter_knowledge}) + + examples = self.build_examples(raw_data) + + def instance_reader(): + """ instance reader """ + for (index, example) in enumerate(examples): + instance = [example[0], example[1], example[2]] + yield instance + + def batch_reader(reader, batch_size): + """ batch reader """ + batch = [] + for instance in reader(): + if len(batch) < batch_size: + batch.append(instance) + else: + yield batch + batch = [instance] + + if len(batch) > 0: + yield batch + + def wrapper(): + """ wrapper """ + for batch in batch_reader(instance_reader, batch_size): + batch_data = self.prepare_batch_data(batch) + yield batch_data + + return wrapper + + def data_generator(self, batch_size, phase, shuffle=False): + """ Generate data for train, dev or test. """ + if phase == 'train': + train_file = os.path.join(self.data_dir, self.data_prefix + ".train") + train_raw = self.read_data(train_file) + examples = self.build_examples(train_raw) + self.num_examples['train'] = len(examples) + elif phase == 'dev': + valid_file = os.path.join(self.data_dir, self.data_prefix + ".dev") + valid_raw = self.read_data(valid_file) + examples = self.build_examples(valid_raw) + self.num_examples['dev'] = len(examples) + elif phase == 'test': + test_file = os.path.join(self.data_dir, self.data_prefix + ".test") + test_raw = self.read_data(test_file) + examples = self.build_examples(test_raw) + self.num_examples['test'] = len(examples) + else: + raise ValueError( + "Unknown phase, which should be in ['train', 'dev', 'test'].") + + def instance_reader(): + """ instance reader """ + if shuffle: + random.shuffle(examples) + for (index, example) in enumerate(examples): + if phase == 'train': + self.current_train_example = index + 1 + instance = [example[0], example[1], example[2]] + yield instance + + def batch_reader(reader, batch_size): + """ batch reader """ + batch = [] + for instance in reader(): + if len(batch) < batch_size: + batch.append(instance) + else: + yield batch + batch = [instance] + + if len(batch) > 0: + yield batch + + def wrapper(): + """ wrapper """ + for batch in batch_reader(instance_reader, batch_size): + batch_data = self.prepare_batch_data(batch) + yield batch_data + + return wrapper + + def prepare_batch_data(self, batch): + """ generate input tensor data """ + batch_source_ids = [inst[0] for inst in batch] + batch_target_ids = [inst[1] for inst in batch] + batch_knowledge_ids = [inst[2] for inst in batch] + + pad_source = max([self.cal_max_len(s_inst) for s_inst in batch_source_ids]) + pad_target = max([self.cal_max_len(t_inst) for t_inst in batch_target_ids]) + pad_kn = max([self.cal_max_len(k_inst) for k_inst in batch_knowledge_ids]) + pad_kn_num = max([len(k_inst) for k_inst in batch_knowledge_ids]) + + source_pad_ids = [self.pad_data(s_inst, pad_source) for s_inst in batch_source_ids] + target_pad_ids = [self.pad_data(t_inst, pad_target) for t_inst in batch_target_ids] + knowledge_pad_ids = [self.pad_data(k_inst, pad_kn, pad_kn_num) + for k_inst in batch_knowledge_ids] + + source_len = [len(inst) for inst in batch_source_ids] + target_len = [len(inst) for inst in batch_target_ids] + kn_len = [[len(term) for term in inst] for inst in batch_knowledge_ids] + kn_len_pad = [] + for elem in kn_len: + if len(elem) < pad_kn_num: + elem += [self.vocab_dict['']] * (pad_kn_num - len(elem)) + kn_len_pad.extend(elem) + + return_array = [np.array(source_pad_ids).reshape(-1, pad_source), np.array(source_len), + np.array(target_pad_ids).reshape(-1, pad_target), np.array(target_len), + np.array(knowledge_pad_ids).astype("int64").reshape(-1, pad_kn_num, pad_kn), + np.array(kn_len_pad).astype("int64").reshape(-1, pad_kn_num)] + + return return_array + + def pad_data(self, insts, pad_len, pad_num=-1): + """ padding ids """ + insts_pad = [] + if isinstance(insts[0], list): + for inst in insts: + inst_pad = inst + [self.vocab_dict['']] * (pad_len - len(inst)) + insts_pad.append(inst_pad) + if len(insts_pad) < pad_num: + insts_pad += [[self.vocab_dict['']] * pad_len] * (pad_num - len(insts_pad)) + else: + insts_pad = insts + [self.vocab_dict['']] * (pad_len - len(insts)) + return insts_pad + + def cal_max_len(self, ids): + """ calculate max sequence length """ + if isinstance(ids[0], list): + pad_len = max([self.cal_max_len(k) for k in ids]) + else: + pad_len = len(ids) + return pad_len diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/models/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/models/__init__.py new file mode 100644 index 00000000..fc0494cb --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/models/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: __init__.py +""" diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/models/knowledge_seq2seq.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/models/knowledge_seq2seq.py new file mode 100644 index 00000000..b8d006da --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/models/knowledge_seq2seq.py @@ -0,0 +1,590 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: source/models/knowledge_seq2seq.py +""" + +import numpy as np +import paddle.fluid as fluid +import paddle.fluid.layers as layers +from paddle.fluid.layers.control_flow import StaticRNN as PaddingRNN +from source.utils.utils import log_softmax + + +def get_embedding(input, emb_size, vocab_size, name=""): + """ get embedding """ + return layers.embedding(input, + size=[vocab_size, emb_size], + param_attr=fluid.ParamAttr(name="embedding"), + is_sparse=True) + + +def fc(input, input_size, output_size, bias=True, name="fc"): + """ fc """ + weight = layers.create_parameter([input_size, output_size], + dtype='float32', + name=name + "_w") + out = layers.matmul(input, weight) + if bias: + bias = layers.create_parameter([output_size], + dtype='float32', + name=name + "_b") + out = out + bias + + return out + + +class GRU_unit(object): + """ GRU unit """ + def __init__(self, input_size, hidden_size, num_layers=1, dropout=0.0, name="gru_unit"): + self.weight_input_array = [] + self.weight_hidden_array = [] + self.bias_input_array = [] + self.bias_hidden_array = [] + self.init_hidden_array = [] + + # init gru param + for i in range(num_layers): + weight_input = layers.create_parameter([input_size, hidden_size * 3], + dtype='float32', + name=name + "_input_w") + self.weight_input_array.append(weight_input) + weight_hidden = layers.create_parameter([hidden_size, hidden_size * 3], + dtype='float32', + name=name + "_hidden_w") + self.weight_hidden_array.append(weight_hidden) + bias_input = layers.create_parameter([hidden_size * 3], + dtype='float32', + name=name + "_input_b") + self.bias_input_array.append(bias_input) + bias_hidden = layers.create_parameter([hidden_size * 3], + dtype='float32', + name=name + "_hidden_b") + self.bias_hidden_array.append(bias_hidden) + + self.dropout = dropout + self.num_layers = num_layers + self.input_size = input_size + self.hidden_size = hidden_size + + def gru_step(self, input, hidden, mask=None): + """ gru step """ + hidden_array = [] + for i in range(self.num_layers): + hidden_temp = layers.slice(hidden, axes = [0], starts = [i], ends = [i + 1]) + hidden_temp = layers.reshape(hidden_temp, shape=[-1, self.hidden_size]) + hidden_array.append(hidden_temp) + + last_hidden_array = [] + for k in range(self.num_layers): + trans_input = layers.matmul(input, self.weight_input_array[k]) + trans_input += self.bias_input_array[k] + trans_hidden = layers.matmul(hidden_array[k], self.weight_hidden_array[k]) + trans_hidden += self.bias_hidden_array[k] + + input_array = layers.split(trans_input, num_or_sections=3, dim=-1) + trans_array = layers.split(trans_hidden, num_or_sections=3, dim=-1) + + reset_gate = layers.sigmoid(input_array[0] + trans_array[0]) + input_gate = layers.sigmoid(input_array[1] + trans_array[1]) + new_gate = layers.tanh(input_array[2] + reset_gate * trans_array[2]) + + new_hidden = new_gate + input_gate * (hidden_array[k] - new_gate) + + if mask: + neg_mask = layers.fill_constant_batch_size_like(input=mask, + shape=[1], + value=1.0, + dtype='float32') - mask + new_hidden = new_hidden * mask + hidden_array[k] * neg_mask + + last_hidden_array.append(new_hidden) + input = new_hidden + + if self.dropout and self.dropout > 0.0: + input = layers.dropout(input, dropout_prob = self.dropout) + + last_hidden = layers.concat(last_hidden_array, 0) + last_hidden = layers.reshape(last_hidden, + shape=[self.num_layers, -1, self.hidden_size]) + + return input, last_hidden + + def __call__(self, input, hidden, mask=None): + return self.gru_step(input, hidden, mask) + + +def gru_rnn(input, input_size, hidden_size, + init_hidden=None, batch_first=False, + mask=None, num_layers=1, dropout=0.0, name="gru"): + """ gru rnn """ + + gru_unit = GRU_unit(input_size, hidden_size, + num_layers=num_layers, dropout=dropout, name=name + "_gru_unit") + + if batch_first: + input = layers.transpose(x=input, perm=[1, 0, 2]) + if mask: + mask = layers.transpose(mask, perm=[1, 0]) + + rnn = PaddingRNN() + with rnn.step(): + step_in = rnn.step_input(input) + step_mask = None + + if mask: + step_mask = rnn.step_input(mask) + + pre_hidden = rnn.memory(init = init_hidden) + new_hidden, last_hidden = gru_unit(step_in, pre_hidden, step_mask) + rnn.update_memory(pre_hidden, last_hidden) + step_in = new_hidden + rnn.step_output(step_in) + rnn.step_output(last_hidden) + + rnn_res = rnn() + rnn_out = rnn_res[0] + last_hidden = layers.slice(rnn_res[1], axes=[0], starts=[-1], ends=[1000000000]) + last_hidden = layers.reshape(last_hidden, shape=[num_layers, -1, hidden_size]) + + if batch_first: + rnnout = layers.transpose(x = rnn_out, perm=[1, 0, 2]) + + return rnnout, last_hidden + + +def bidirec_gru(input, input_size, hidden_size, + batch_size, batch_first=True, num_layers=1, + dropout=0.0, mask=None, last_mask=None, name='bidir_gru'): + """ bidirec gru """ + + # use lod dynamic gru + def gru_fun(gru_in, name=None, is_reverse=False): + """ gru fun """ + fw_last_array = [] + fw_in = gru_in + for i in range(num_layers): + fw_gru_in = layers.fc(input=fw_in, size=hidden_size * 3, + param_attr=fluid.ParamAttr(name=name + "_fc_w"), + bias_attr=fluid.ParamAttr(name=name + "_fc_b")) + fw_gru_out = layers.dynamic_gru(input=fw_gru_in, size=hidden_size, + param_attr= fluid.ParamAttr(name=name + "_w"), + bias_attr=fluid.ParamAttr(name=name + "_b"), + origin_mode=True, is_reverse=is_reverse) + fw_in = fw_gru_out + + if is_reverse: + fw_last_hidden = layers.sequence_first_step(fw_gru_out) + else: + fw_last_hidden = layers.sequence_last_step(fw_gru_out) + + if last_mask: + fw_last_hidden = layers.elementwise_mul(fw_last_hidden, last_mask, axis=0) + + fw_last_array.append(fw_last_hidden) + + if num_layers == 1: + final_fw_last_hidden = layers.unsqueeze(fw_last_array[0], axes=[0]) + else: + final_fw_last_hidden = layers.concat(fw_last_array, axis=0) + final_fw_last_hidden = layers.reshape(final_fw_last_hidden, + shape=[num_layers, -1, hidden_size]) + + final_fw_out = fw_in + return final_fw_out, final_fw_last_hidden + + fw_rnn_out, fw_last_hidden = gru_fun(input, name=name + "_fw") + bw_rnn_out, bw_last_hidden = gru_fun(input, name=name + "_bw", is_reverse=True) + + return [fw_rnn_out, bw_rnn_out, fw_last_hidden, bw_last_hidden] + + +def dot_attention(query, memory, mask=None): + """ dot attention """ + attn = layers.matmul(query, memory, transpose_y=True) + + if mask: + attn += mask * -1000000000 + + weight = layers.softmax(attn) + weight_memory = layers.matmul(weight, memory) + + return weight_memory, weight + + +def rnn_encoder(input, vocab_size, input_size, hidden_size, + batch_size, num_layers, bi_direc, dropout=0.0, + batch_first=True, mask=None, last_mask=None, name="rnn_enc"): + """ rnn encoder """ + input_emb = get_embedding(input, input_size, vocab_size) + fw_rnn_out, bw_rnn_out, fw_last_hidden, bw_last_hidden = \ + bidirec_gru(input_emb, input_size, hidden_size, batch_size, + batch_first=batch_first, num_layers = num_layers, + dropout=dropout, mask=mask, last_mask = last_mask, name=name) + + output = layers.concat([fw_rnn_out, bw_rnn_out], axis = 1) + last_hidden = layers.concat([fw_last_hidden, bw_last_hidden], axis= 2) + + return output, last_hidden + + +def decoder_step(gru_unit, cue_gru_unit, step_in, + hidden, input_size, hidden_size, + memory, memory_mask, knowledge, mask=None): + """ decoder step """ + # get attention out + # get hidden top layers + top_hidden = layers.slice(hidden, axes=[0], starts=[0], ends=[1]) + top_hidden = layers.squeeze(top_hidden, axes=[0]) + top_hidden = layers.unsqueeze(top_hidden, axes=[1]) + + weight_memory, attn = dot_attention(top_hidden, memory, memory_mask) + + step_in = layers.unsqueeze(step_in, axes=[1]) + rnn_input_list = [step_in, weight_memory] + if weight_memory.shape[0] == -1: + knowledge_1 = layers.reshape(knowledge, shape=weight_memory.shape) + else: + knowledge_1 = knowledge + cue_input_list = [knowledge_1, weight_memory] + output_list = [weight_memory] + + rnn_input = layers.concat(rnn_input_list, axis=2) + + rnn_input = layers.squeeze(rnn_input, axes=[1]) + rnn_output, rnn_last_hidden = gru_unit(rnn_input, hidden, mask) + + cue_input = layers.concat(cue_input_list, axis=2) + cue_input = layers.squeeze(cue_input, axes=[1]) + cue_rnn_out, cue_rnn_last_hidden = cue_gru_unit(cue_input, hidden, mask) + + h_y = layers.tanh(fc(rnn_last_hidden, hidden_size, hidden_size, name="dec_fc1")) + h_cue = layers.tanh(fc(cue_rnn_last_hidden, hidden_size, hidden_size, name="dec_fc2")) + + concate_y_cue = layers.concat([h_y, h_cue], axis=2) + k = layers.sigmoid(fc(concate_y_cue, hidden_size * 2, 1, name='dec_fc3')) + + new_hidden = h_y * k - h_cue * (k - 1.0) + + new_hidden_tmp = layers.transpose(new_hidden, perm=[1, 0, 2]) + output_list.append(new_hidden_tmp) + + real_out = layers.concat(output_list, axis=2) + + if mask: + mask_tmp = layers.unsqueeze(mask, axes=[0]) + new_hidden = layers.elementwise_mul((new_hidden - hidden), mask_tmp, axis=0) + new_hidden += hidden + + return real_out, new_hidden + + +def rnn_decoder(gru_unit, cue_gru_unit, input, input_size, hidden_size, + num_layers, memory, memory_mask, knowledge, output_size, + init_hidden=None, mask=None, dropout=0.0, batch_first=True, name="decoder"): + """ rnn decoder """ + input_emb = get_embedding(input, input_size, output_size) + if batch_first: + input_emb = layers.transpose(input_emb, perm=[1, 0, 2]) + if mask: + trans_mask = layers.transpose(mask, perm=[1, 0]) + + rnn = PaddingRNN() + with rnn.step(): + step_in = rnn.step_input(input_emb) + step_mask = None + + if mask: + step_mask = rnn.step_input(trans_mask) + + # split pre_hidden + pre_hidden_list = [] + + pre_hidden = rnn.memory(init = init_hidden) + real_out, last_hidden = \ + decoder_step(gru_unit, cue_gru_unit, step_in, pre_hidden, input_size, + hidden_size, memory, memory_mask, knowledge, mask=step_mask) + + rnn.update_memory(pre_hidden, last_hidden) + + step_in = layers.squeeze(real_out, axes=[1]) + rnn.step_output(step_in) + + rnnout = rnn() + rnnout = layers.transpose(rnnout, perm=[1, 0, 2]) + rnnout = layers.elementwise_mul(rnnout, mask, axis=0) + + output_in_size = hidden_size + hidden_size + rnnout = layers.dropout(rnnout, dropout_prob = dropout) + rnnout = fc(rnnout, output_in_size, hidden_size, name='dec_out_fc1') + rnnout = fc(rnnout, hidden_size, output_size, name='dec_out_fc2') + + softmax_out = layers.softmax(rnnout) + + return softmax_out + + +def knowledge_seq2seq(config): + """ knowledge seq2seq """ + emb_size = config.embed_size + hidden_size = config.hidden_size + input_size = emb_size + num_layers = config.num_layers + bi_direc = config.bidirectional + batch_size = config.batch_size + vocab_size = config.vocab_size + run_type = config.run_type + + enc_input = layers.data(name="enc_input", shape=[1], dtype='int64', lod_level=1) + enc_mask = layers.data(name="enc_mask", shape=[-1, 1], dtype='float32') + cue_input = layers.data(name="cue_input", shape=[1], dtype='int64', lod_level=1) + #cue_mask = layers.data(name='cue_mask', shape=[-1, 1], dtype='float32') + memory_mask = layers.data(name='memory_mask', shape=[-1, 1], dtype='float32') + tar_input = layers.data(name='tar_input', shape=[1], dtype='int64', lod_level=1) + # tar_mask = layers.data(name="tar_mask", shape=[-1, 1], dtype='float32') + + rnn_hidden_size = hidden_size + if bi_direc: + rnn_hidden_size //= 2 + + enc_out, enc_last_hidden = \ + rnn_encoder(enc_input, vocab_size, input_size, rnn_hidden_size, batch_size, num_layers, bi_direc, + dropout=0.0, batch_first=True, name="rnn_enc") + + bridge_out = fc(enc_last_hidden, hidden_size, hidden_size, name="bridge") + bridge_out = layers.tanh(bridge_out) + + cue_last_mask = layers.data(name='cue_last_mask', shape=[-1], dtype='float32') + knowledge_out, knowledge_last_hidden = \ + rnn_encoder(cue_input, vocab_size, input_size, rnn_hidden_size, batch_size, num_layers, bi_direc, + dropout=0.0, batch_first=True, last_mask=cue_last_mask, name="knowledge_enc") + + query = layers.slice(bridge_out, axes=[0], starts=[0], ends=[1]) + query = layers.squeeze(query, axes=[0]) + query = layers.unsqueeze(query, axes=[1]) + query = layers.reshape(query, shape=[batch_size, -1, hidden_size]) + cue_memory = layers.slice(knowledge_last_hidden, axes=[0], starts=[0], ends=[1]) + cue_memory = layers.reshape(cue_memory, shape=[batch_size, -1, hidden_size]) + memory_mask = layers.reshape(memory_mask, shape=[batch_size, 1, -1]) + + weighted_cue, cue_att = dot_attention(query, cue_memory, mask=memory_mask) + + cue_att = layers.reshape(cue_att, shape=[batch_size, -1]) + + knowledge = weighted_cue + if config.use_posterior: + target_out, target_last_hidden = \ + rnn_encoder(tar_input, vocab_size, input_size, rnn_hidden_size, batch_size, num_layers, bi_direc, + dropout=0.0, batch_first=True, name="knowledge_enc") + + # get attenion + target_query = layers.slice(target_last_hidden, axes=[0], starts=[0], ends=[1]) + target_query = layers.squeeze(target_query, axes=[0]) + target_query = layers.unsqueeze(target_query, axes=[1]) + target_query = layers.reshape(target_query, shape=[batch_size, -1, hidden_size]) + + weight_target, target_att = dot_attention(target_query, cue_memory, mask=memory_mask) + target_att = layers.reshape(target_att, shape=[batch_size, -1]) + # add to output + knowledge = weight_target + + enc_memory_mask = layers.data(name="enc_memory_mask", shape=[-1, 1], dtype='float32') + enc_memory_mask = layers.unsqueeze(enc_memory_mask, axes=[1]) + # decoder init_hidden, enc_memory, enc_mask + dec_init_hidden = bridge_out + pad_value = fluid.layers.assign( + input=np.array([0.0], dtype='float32')) + + enc_memory, origl_len_1 = layers.sequence_pad(x = enc_out, pad_value=pad_value) + enc_memory.persistable = True + + gru_unit = GRU_unit(input_size + hidden_size, hidden_size, + num_layers=num_layers, dropout=0.0, name="decoder_gru_unit") + + cue_gru_unit = GRU_unit(hidden_size + hidden_size, hidden_size, + num_layers=num_layers, dropout=0.0, name="decoder_cue_gru_unit") + + tgt_vocab_size = config.vocab_size + if run_type == "train": + if config.use_bow: + bow_logits = fc(knowledge, hidden_size, hidden_size, name='bow_fc_1') + bow_logits = layers.tanh(bow_logits) + bow_logits = fc(bow_logits, hidden_size, tgt_vocab_size, name='bow_fc_2') + bow_logits = layers.softmax(bow_logits) + + bow_label = layers.data(name='bow_label', shape=[-1, config.max_len], dtype='int64') + bow_mask = layers.data(name="bow_mask", shape=[-1, config.max_len], dtype='float32') + + bow_logits = layers.expand(bow_logits, [1, config.max_len, 1]) + bow_logits = layers.reshape(bow_logits, shape=[-1, tgt_vocab_size]) + bow_label = layers.reshape(bow_label, shape=[-1, 1]) + bow_loss = layers.cross_entropy(bow_logits, bow_label, soft_label=False) + bow_loss = layers.reshape(bow_loss, shape=[-1, config.max_len]) + + bow_loss *= bow_mask + bow_loss = layers.reduce_sum(bow_loss, dim=[1]) + bow_loss = layers.reduce_mean(bow_loss) + + dec_input = layers.data(name="dec_input", shape=[-1, 1, 1], dtype='int64') + dec_mask = layers.data(name="dec_mask", shape=[-1, 1], dtype='float32') + + dec_knowledge = weight_target + + decoder_logits = \ + rnn_decoder(gru_unit, cue_gru_unit, dec_input, input_size, hidden_size, num_layers, + enc_memory, enc_memory_mask, dec_knowledge, vocab_size, + init_hidden=dec_init_hidden, mask=dec_mask, dropout=config.dropout) + + target_label = layers.data(name='target_label', shape=[-1, 1], dtype='int64') + target_mask = layers.data(name='target_mask', shape=[-1, 1], dtype='float32') + + decoder_logits = layers.reshape(decoder_logits, shape=[-1, tgt_vocab_size]) + target_label = layers.reshape(target_label, shape=[-1, 1]) + + nll_loss = layers.cross_entropy(decoder_logits, target_label, soft_label = False) + nll_loss = layers.reshape(nll_loss, shape=[batch_size, -1]) + nll_loss *= target_mask + nll_loss = layers.reduce_sum(nll_loss, dim=[1]) + nll_loss = layers.reduce_mean(nll_loss) + + prior_attn = cue_att + 1e-10 + posterior_att = target_att + posterior_att.stop_gradient = True + + prior_attn = layers.log(prior_attn) + + kl_loss = posterior_att * (layers.log(posterior_att + 1e-10) - prior_attn) + kl_loss = layers.reduce_mean(kl_loss) + + kl_and_nll_factor = layers.data(name='kl_and_nll_factor', shape=[1], dtype='float32') + kl_and_nll_factor = layers.reshape(kl_and_nll_factor, shape=[-1]) + + + final_loss = bow_loss + kl_loss * kl_and_nll_factor + nll_loss * kl_and_nll_factor + + return [bow_loss, kl_loss, nll_loss, final_loss] + + elif run_type == "test": + beam_size = config.beam_size + batch_size = config.batch_size + token = layers.fill_constant(shape=[batch_size * beam_size, 1], + value=config.bos_id, dtype='int64') + + token = layers.reshape(token, shape=[-1, 1]) + max_decode_len = config.max_dec_len + + dec_knowledge = knowledge + INF= 100000000.0 + + init_score_np = np.ones([beam_size * batch_size], dtype='float32') * -INF + + for i in range(batch_size): + init_score_np[i * beam_size] = 0.0 + + pre_score = layers.assign(init_score_np) + + pos_index_np = np.arange(batch_size).reshape(-1, 1) + pos_index_np = \ + np.tile(pos_index_np, (1, beam_size)).reshape(-1).astype('int32') * beam_size + + pos_index = layers.assign(pos_index_np) + + id_array = [] + score_array = [] + index_array = [] + init_enc_memory = layers.expand(enc_memory, [1, beam_size, 1]) + init_enc_memory = layers.reshape(init_enc_memory, + shape=[batch_size * beam_size, -1, hidden_size]) + init_enc_mask = layers.expand(enc_memory_mask, [1, beam_size, 1]) + init_enc_mask = layers.reshape(init_enc_mask, shape=[batch_size * beam_size, 1, -1]) + + dec_knowledge = layers.reshape(dec_knowledge, shape=[-1, 1, hidden_size]) + init_dec_knowledge = layers.expand(dec_knowledge, [1, beam_size, 1]) + init_dec_knowledge = layers.reshape(init_dec_knowledge, + shape=[batch_size * beam_size, -1, hidden_size]) + + dec_init_hidden = layers.expand(dec_init_hidden, [1, 1, beam_size]) + dec_init_hidden = layers.reshape(dec_init_hidden, shape=[1, -1, hidden_size]) + + length_average = config.length_average + UNK = config.unk_id + EOS = config.eos_id + for i in range(1, max_decode_len + 1): + dec_emb = get_embedding(token, input_size, vocab_size) + dec_out, dec_last_hidden = \ + decoder_step(gru_unit, cue_gru_unit, + dec_emb, dec_init_hidden, input_size, hidden_size, + init_enc_memory, init_enc_mask, init_dec_knowledge, mask=None) + output_in_size = hidden_size + hidden_size + + rnnout = layers.dropout(dec_out, dropout_prob=config.dropout, is_test = True) + rnnout = fc(rnnout, output_in_size, hidden_size, name='dec_out_fc1') + rnnout = fc(rnnout, hidden_size, vocab_size, name='dec_out_fc2') + + log_softmax_output = log_softmax(rnnout) + log_softmax_output = layers.squeeze(log_softmax_output, axes=[1]) + + if i > 1: + if length_average: + log_softmax_output = layers.elementwise_add((log_softmax_output / i), + (pre_score * (1.0 - 1.0 / i)), + axis=0) + else: + log_softmax_output = layers.elementwise_add(log_softmax_output, + pre_score, axis=0) + else: + log_softmax_output = layers.elementwise_add(log_softmax_output, + pre_score, axis=0) + + log_softmax_output = layers.reshape(log_softmax_output, shape=[batch_size, -1]) + + topk_score, topk_index = layers.topk(log_softmax_output, k = beam_size) + topk_score = layers.reshape(topk_score, shape=[-1]) + topk_index = layers.reshape(topk_index, shape =[-1]) + + vocab_var = layers.fill_constant([1], dtype='int64', value=vocab_size) + new_token = topk_index % vocab_var + + index = topk_index // vocab_var + id_array.append(new_token) + index_array.append(index) + index = index + pos_index + + score_array.append(topk_score) + + eos_ids = layers.fill_constant([beam_size * batch_size], dtype='int64', value=EOS) + unk_ids = layers.fill_constant([beam_size * batch_size], dtype='int64', value=UNK) + eos_eq = layers.cast(layers.equal(new_token, eos_ids), dtype='float32') + + topk_score += eos_eq * -100000000.0 + + unk_eq = layers.cast(layers.equal(new_token, unk_ids), dtype='float32') + topk_score += unk_eq * -100000000.0 + + # update + token = new_token + pre_score = topk_score + token = layers.reshape(token, shape=[-1, 1]) + + index = layers.cast(index, dtype='int32') + dec_last_hidden = layers.squeeze(dec_last_hidden, axes=[0]) + dec_init_hidden = layers.gather(dec_last_hidden, index=index) + dec_init_hidden = layers.unsqueeze(dec_init_hidden, axes=[0]) + init_enc_memory = layers.gather(init_enc_memory, index) + init_enc_mask = layers.gather(init_enc_mask, index) + init_dec_knowledge = layers.gather(init_dec_knowledge, index) + + final_score = layers.concat(score_array, axis=0) + final_ids = layers.concat(id_array, axis=0) + final_index = layers.concat(index_array, axis = 0) + + final_score = layers.reshape(final_score, shape=[max_decode_len, beam_size * batch_size]) + final_ids = layers.reshape(final_ids, shape=[max_decode_len, beam_size * batch_size]) + final_index = layers.reshape(final_index, shape=[max_decode_len, beam_size * batch_size]) + + return final_score, final_ids, final_index diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/utils/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/utils/__init__.py new file mode 100644 index 00000000..c97406f8 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/utils/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: __init__.py +""" \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/utils/utils.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/utils/utils.py new file mode 100644 index 00000000..a7228f43 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/source/utils/utils.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: source/utils/utils.py +""" + +import argparse +import numpy as np +import paddle.fluid as fluid +import paddle.fluid.layers as layers + + +def str2bool(v): + """ str2bool """ + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + +def load_id2str_dict(vocab_file): + """ load id2str dict """ + id_dict_array = [] + with open(vocab_file, 'r') as fr: + for line in fr: + line = line.strip() + id_dict_array.append(line) + + return id_dict_array + + +def load_str2id_dict(vocab_file): + """ load str2id dict """ + words_dict = {} + with open(vocab_file, 'r') as fr: + for line in fr: + word = line.strip() + words_dict[word] = len(words_dict) + + return words_dict + + +def log_softmax(x): + """ log softmax """ + t1 = layers.exp(x) + t1 = layers.reduce_sum(t1, dim=-1) + t1 = layers.log(t1) + return layers.elementwise_sub(x, t1, axis=0) + + +def id_to_text(ids, id_dict_array): + """ convert id seq to str seq """ + res = [] + for i in ids: + res.append(id_dict_array[i]) + + return ' '.join(res) + + +def pad_to_bath_size(src_ids, src_len, trg_ids, trg_len, kn_ids, kn_len, batch_size): + """ pad to bath size for knowledge corpus""" + real_len = src_ids.shape[0] + + def pad(old): + """ pad """ + old_shape = list(old.shape) + old_shape[0] = batch_size + new_val = np.zeros(old_shape, dtype=old.dtype) + new_val[:real_len] = old + for i in range(real_len, batch_size): + new_val[i] = old[-1] + return new_val + + new_src_ids = pad(src_ids) + new_src_len = pad(src_len) + new_trg_ids = pad(trg_ids) + new_trg_len = pad(trg_len) + new_kn_ids = pad(kn_ids) + new_kn_len = pad(kn_len) + + return [new_src_ids, new_src_len, new_trg_ids, new_trg_len, new_kn_ids, new_kn_len] + + +def to_lodtensor(data, seq_lens, place): + """ convert to LoDTensor """ + cur_len = 0 + lod = [cur_len] + + data_array = [] + for idx, seq in enumerate(seq_lens): + if seq > 0: + data_array.append(data[idx, :seq]) + + cur_len += seq + lod.append(cur_len) + else: + data_array.append(np.zeros([1, 1], dtype='int64')) + cur_len += 1 + lod.append(cur_len) + flattened_data = np.concatenate(data_array, axis=0).astype("int64") + flattened_data = flattened_data.reshape([len(flattened_data), 1]) + res = fluid.LoDTensor() + res.set(flattened_data, place) + + res.set_lod([lod]) + return res + + +def len_to_mask(len_seq, max_len=None): + """ len to mask """ + if max_len is None: + max_len = np.max(len_seq) + + mask = np.zeros((len_seq.shape[0], max_len), dtype='float32') + + for i, l in enumerate(len_seq): + mask[i, :l] = 1.0 + + return mask + + +def build_data_feed(data, place, + batch_size=128, + is_training=False, + bow_max_len=30, + pretrain_epoch=False): + """ build data feed """ + src_ids, src_len, trg_ids, trg_len, kn_ids, kn_len = data + + real_size = src_ids.shape[0] + if src_ids.shape[0] < batch_size: + if not is_training: + src_ids, src_len, trg_ids, trg_len, kn_ids, kn_len = \ + pad_to_bath_size(src_ids, src_len, trg_ids, trg_len, kn_ids, kn_len, batch_size) + else: + return None + + enc_input = np.expand_dims(src_ids[:, 1: -1], axis=2) + enc_mask = len_to_mask(src_len - 2) + + tar_input = np.expand_dims(trg_ids[:, 1: -1], axis=2) + tar_mask = len_to_mask(trg_len - 2) + cue_input = np.expand_dims(kn_ids.reshape((-1, kn_ids.shape[-1]))[:, 1:-1], axis=2) + cue_mask = len_to_mask(kn_len.reshape(-1) - 2) + memory_mask = np.equal(kn_len, 0).astype('float32') + + enc_memory_mask = 1.0 - enc_mask + + if not is_training: + return {'enc_input': to_lodtensor(enc_input, src_len - 2, place), + 'enc_mask': enc_mask, + 'cue_input': to_lodtensor(cue_input, kn_len.reshape(-1) - 2, place), + 'cue_last_mask': np.not_equal(kn_len.reshape(-1), 0).astype('float32'), + 'memory_mask': memory_mask, + 'enc_memory_mask': enc_memory_mask, + }, real_size + + dec_input = np.expand_dims(trg_ids[:, :-1], axis=2) + dec_mask = len_to_mask(trg_len - 1) + + target_label = trg_ids[:, 1:] + target_mask = len_to_mask(trg_len - 1) + + bow_label = target_label[:, :-1] + bow_label = np.pad(bow_label, ((0, 0), (0, bow_max_len - bow_label.shape[1])), 'constant', constant_values=(0)) + bow_mask = np.pad(np.not_equal(bow_label, 0).astype('float32'), ((0, 0), (0, bow_max_len - bow_label.shape[1])), + 'constant', constant_values=(0.0)) + + if not pretrain_epoch: + kl_and_nll_factor = np.ones([1], dtype='float32') + else: + kl_and_nll_factor = np.zeros([1], dtype='float32') + + return {'enc_input': to_lodtensor(enc_input, src_len - 2, place), + 'enc_mask': enc_mask, + 'cue_input': to_lodtensor(cue_input, kn_len.reshape(-1) - 2, place), + 'cue_last_mask': np.not_equal(kn_len.reshape(-1), 0).astype('float32'), + 'memory_mask': memory_mask, + 'enc_memory_mask': enc_memory_mask, + 'tar_input': to_lodtensor(tar_input, trg_len - 2, place), + 'bow_label': bow_label, + 'bow_mask': bow_mask, + 'target_label': target_label, + 'target_mask': target_mask, + 'dec_input': dec_input, + 'dec_mask': dec_mask, + 'kl_and_nll_factor': kl_and_nll_factor} + + +def load_embedding(embedding_file, vocab_file): + """ load pretrain embedding from file """ + words_dict = load_str2id_dict(vocab_file) + coverage = 0 + print("Building word embeddings from '{}' ...".format(embedding_file)) + with open(embedding_file, "r") as f: + num, dim = map(int, f.readline().strip().split()) + embeds = [[0] * dim] * len(words_dict) + for line in f: + w, vs = line.rstrip().split(" ", 1) + if w in words_dict: + try: + vs = [float(x) for x in vs.split(" ")] + except Exception: + vs = [] + if len(vs) == dim: + embeds[words_dict[w]] = vs + coverage += 1 + rate = coverage * 1.0 / len(embeds) + print("{} words have pretrained {}-D word embeddings (coverage: {:.3f})".format( \ + coverage, dim, rate)) + + return np.array(embeds).astype('float32') + + +def init_embedding(embedding_file, vocab_file, init_scale, shape): + """ init embedding by pretrain file or random """ + if embedding_file != "": + try: + emb_np = load_embedding(embedding_file, vocab_file) + except: + print("load init emb file failed", embedding_file) + raise Exception("load embedding file failed") + + if emb_np.shape != shape: + print("shape not match", emb_np.shape, shape) + raise Exception("shape not match") + + zero_count = 0 + for i in range(emb_np.shape[0]): + if np.sum(emb_np[i]) == 0: + zero_count += 1 + emb_np[i] = np.random.uniform(-init_scale, init_scale, emb_np.shape[1:]).astype('float32') + else: + print("random init embeding") + emb_np = np.random.uniform(-init_scale, init_scale, shape).astype('float32') + + return emb_np diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/__init__.py new file mode 100755 index 00000000..c97406f8 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: __init__.py +""" \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/build_vocabulary.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/build_vocabulary.py new file mode 100755 index 00000000..ac194e3e --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/build_vocabulary.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: build_vocabulary.py +""" + +from __future__ import print_function + +import sys +import re +from collections import Counter + +reload(sys) +sys.setdefaultencoding('utf8') + + +def tokenize(s): + """ + tokenize + """ + s = re.sub('\d+', '', s).lower() + tokens = s.split(' ') + return tokens + + +def build_vocabulary(corpus_file, vocab_file, + vocab_size=30004, min_frequency=0, + min_len=1, max_len=500): + """ + build words dict + """ + specials = ["", "", "", ""] + counter = Counter() + for line in open(corpus_file, 'r'): + src, tgt, knowledge = line.rstrip('\n').split('\t')[:3] + filter_knowledge = [] + for sent in knowledge.split('\1'): + filter_knowledge.append(' '.join(sent.split()[:max_len])) + knowledge = ' '.join(filter_knowledge) + + src = tokenize(src) + tgt = tokenize(tgt) + knowledge = tokenize(knowledge) + + if len(src) < min_len or len(src) > max_len or \ + len(tgt) < min_len or len(tgt) > max_len: + continue + + counter.update(src + tgt + knowledge) + + for tok in specials: + del counter[tok] + + words_and_frequencies = sorted(counter.items(), key=lambda tup: tup[0]) + words_and_frequencies.sort(key=lambda tup: tup[1], reverse=True) + words_and_frequencies = [[tok, sys.maxint] for tok in specials] + words_and_frequencies + words_and_frequencies = words_and_frequencies[:vocab_size] + + fout = open(vocab_file, 'w') + for word, frequency in words_and_frequencies: + if frequency < min_frequency: + break + fout.write(word + '\n') + + fout.close() + + +def main(): + """ + main + """ + if len(sys.argv) < 3: + print("Usage: " + sys.argv[0] + " corpus_file vocab_file") + exit() + + build_vocabulary(sys.argv[1], sys.argv[2]) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_client.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_client.py new file mode 100755 index 00000000..28ddeb49 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_client.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: conversation_client.py +""" + +from __future__ import print_function + +import sys +import socket + +reload(sys) +sys.setdefaultencoding('utf8') + +SERVER_IP = "127.0.0.1" +SERVER_PORT = 8601 + +def conversation_client(text): + """ + conversation_client + """ + mysocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + mysocket.connect((SERVER_IP, SERVER_PORT)) + + mysocket.sendall(text.encode()) + result = mysocket.recv(4096).decode() + + mysocket.close() + + return result + + +def main(): + """ + main + """ + if len(sys.argv) < 2: + print("Usage: " + sys.argv[0] + " eval_file") + exit() + + for line in open(sys.argv[1]): + response = conversation_client(line.strip()) + print(response) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_server.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_server.py new file mode 100755 index 00000000..cc6a076b --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_server.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: conversation_server.py +""" + +from __future__ import print_function + +import sys +sys.path.append("../") +import socket +from thread import start_new_thread +from tools.conversation_strategy import load +from tools.conversation_strategy import predict + +reload(sys) +sys.setdefaultencoding('utf8') + +SERVER_IP = "127.0.0.1" +SERVER_PORT = 8601 + +print("starting conversation server ...") +print("binding socket ...") +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +#Bind socket to local host and port +try: + s.bind((SERVER_IP, SERVER_PORT)) +except socket.error as msg: + print("Bind failed. Error Code : " + str(msg[0]) + " Message " + msg[1]) + exit() +#Start listening on socket +s.listen(10) +print("bind socket success !") + +print("loading model...") +model = load() +print("load model success !") + +print("start conversation server success !") + + +def clientthread(conn, addr): + """ + client thread + """ + logstr = "addr:" + addr[0]+ "_" + str(addr[1]) + try: + #Receiving from client + param = conn.recv(4096).decode() + logstr += "\tparam:" + param + if param is not None: + response = predict(model, param.strip()) + logstr += "\tresponse:" + response + conn.sendall(response.encode()) + conn.close() + print(logstr + "\n") + except Exception as e: + print(logstr + "\n", e) + + +while True: + conn, addr = s.accept() + start_new_thread(clientthread, (conn, addr)) +s.close() diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_strategy.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_strategy.py new file mode 100755 index 00000000..b15cd2f5 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/conversation_strategy.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: conversation_strategy.py +""" + +from __future__ import print_function + +import sys + +sys.path.append("../") +import network +from tools.convert_conversation_corpus_to_model_text import preprocessing_for_one_conversation + +reload(sys) +sys.setdefaultencoding('utf8') + + +def load(): + """ + load model + """ + return network.load() + + +def predict(model, text): + """ + predict + """ + model_text, topic_dict = \ + preprocessing_for_one_conversation(text.strip(), topic_generalization=True) + + if isinstance(model_text, unicode): + model_text = model_text.encode('utf-8') + + response = network.predict(model, model_text) + + topic_list = sorted(topic_dict.items(), key=lambda item: len(item[1]), reverse=True) + for key, value in topic_list: + response = response.replace(key, value) + + return response + + +def main(): + """ + main + """ + generator = load() + for line in sys.stdin: + response = predict(generator, line.strip()) + print(response) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_conversation_corpus_to_model_text.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_conversation_corpus_to_model_text.py new file mode 100755 index 00000000..045f5748 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_conversation_corpus_to_model_text.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: convert_conversation_corpus_to_model_text.py +""" + +from __future__ import print_function + +import sys +import json +import collections + +reload(sys) +sys.setdefaultencoding('utf8') + + +def preprocessing_for_one_conversation(text, + topic_generalization=False): + """ + preprocessing_for_one_conversation + """ + conversation = json.loads(text.strip(), encoding="utf-8", \ + object_pairs_hook=collections.OrderedDict) + + goal = conversation["goal"] + knowledge = conversation["knowledge"] + history = conversation["history"] + response = conversation["response"] if "response" in conversation else "null" + + topic_a = goal[0][1] + topic_b = goal[0][2] + for i, [s, p, o] in enumerate(knowledge): + if u"领域" == p: + if topic_a == s: + domain_a = o + elif topic_b == s: + domain_b = o + + topic_dict = {} + if u"电影" == domain_a: + topic_dict["video_topic_a"] = topic_a + else: + topic_dict["person_topic_a"] = topic_a + + if u"电影" == domain_b: + topic_dict["video_topic_b"] = topic_b + else: + topic_dict["person_topic_b"] = topic_b + + chat_path_str = ' '.join([' '.join(spo) for spo in goal]) + knowledge_str1 = ' '.join([' '.join(spo) for spo in knowledge]) + knowledge_str2 = '\1'.join([' '.join(spo) for spo in knowledge]) + history_str = ' '.join(history) + + src = chat_path_str + " " + knowledge_str1 + " : " + history_str + model_text = '\t'.join([src, response, knowledge_str2]) + + if topic_generalization: + topic_list = sorted(topic_dict.items(), key=lambda item: len(item[1]), reverse=True) + for key, value in topic_list: + model_text = model_text.replace(value, key) + + return model_text, topic_dict + + +def convert_conversation_corpus_to_model_text(corpus_file, text_file, topic_file, \ + topic_generalization=False): + """ + convert_conversation_corpus_to_model_text + """ + fout_text = open(text_file, 'w') + fout_topic = open(topic_file, 'w') + with open(corpus_file, 'r') as f: + for i, line in enumerate(f): + model_text, topic_dict = preprocessing_for_one_conversation( + line.strip(), topic_generalization=topic_generalization) + + topic_dict = json.dumps(topic_dict, ensure_ascii=False, encoding="utf-8") + + fout_text.write(model_text + "\n") + fout_topic.write(topic_dict + "\n") + + fout_text.close() + fout_topic.close() + + +def main(): + """ + main + """ + convert_conversation_corpus_to_model_text(sys.argv[1], + sys.argv[2], + sys.argv[3], + int(sys.argv[4]) > 0) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_result_for_eval.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_result_for_eval.py new file mode 100755 index 00000000..d5a2405b --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_result_for_eval.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: convert_result_for_eval.py +""" + +from __future__ import print_function + +import sys +import json +import collections + +reload(sys) +sys.setdefaultencoding('utf8') + + +def convert_result_for_eval(sample_file, result_file, output_file): + """ + convert_result_for_eval + """ + sample_list = [line.strip() for line in open(sample_file, 'r')] + result_list = [line.strip() for line in open(result_file, 'r')] + + assert len(sample_list) == len(result_list) + fout = open(output_file, 'w') + for i, sample in enumerate(sample_list): + sample = json.loads(sample, encoding="utf-8", \ + object_pairs_hook=collections.OrderedDict) + response = sample["response"] + fout.write(result_list[i] + "\t" + response + "\n") + + fout.close() + + +def main(): + """ + main + """ + convert_result_for_eval(sys.argv[1], + sys.argv[2], + sys.argv[3]) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_session_to_sample.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_session_to_sample.py new file mode 100755 index 00000000..02b83a3c --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/convert_session_to_sample.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: convert_session_to_sample.py +""" + +from __future__ import print_function + +import sys +import json +import collections + +reload(sys) +sys.setdefaultencoding('utf8') + + +def convert_session_to_sample(session_file, sample_file): + """ + convert_session_to_sample + """ + fout = open(sample_file, 'w') + with open(session_file, 'r') as f: + for i, line in enumerate(f): + session = json.loads(line.strip(), encoding="utf-8", \ + object_pairs_hook=collections.OrderedDict) + conversation = session["conversation"] + + for j in range(0, len(conversation), 2): + sample = collections.OrderedDict() + sample["goal"] = session["goal"] + sample["knowledge"] = session["knowledge"] + sample["history"] = conversation[:j] + sample["response"] = conversation[j] + + sample = json.dumps(sample, ensure_ascii=False, encoding="utf-8") + + fout.write(sample + "\n") + + fout.close() + + +def main(): + """ + main + """ + convert_session_to_sample(sys.argv[1], sys.argv[2]) + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/eval.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/eval.py new file mode 100755 index 00000000..c0ec9485 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/eval.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: eval.py +""" + +from __future__ import print_function + +import sys +import math +from collections import Counter + +reload(sys) +sys.setdefaultencoding('utf8') + + +if len(sys.argv) < 2: + print("Usage: " + sys.argv[0] + " eval_file") + print("eval file format: pred_response \t gold_response") + exit() + +def get_dict(tokens, ngram, gdict=None): + """ + get_dict + """ + token_dict = {} + if gdict is not None: + token_dict = gdict + tlen = len(tokens) + for i in range(0, tlen - ngram + 1): + ngram_token = "".join(tokens[i:(i + ngram)]) + if token_dict.get(ngram_token) is not None: + token_dict[ngram_token] += 1 + else: + token_dict[ngram_token] = 1 + return token_dict + + +def count(pred_tokens, gold_tokens, ngram, result): + """ + count + """ + cover_count, total_count = result + pred_dict = get_dict(pred_tokens, ngram) + gold_dict = get_dict(gold_tokens, ngram) + cur_cover_count = 0 + cur_total_count = 0 + for token, freq in pred_dict.items(): + if gold_dict.get(token) is not None: + gold_freq = gold_dict[token] + cur_cover_count += min(freq, gold_freq) + cur_total_count += freq + result[0] += cur_cover_count + result[1] += cur_total_count + + +def calc_bp(pair_list): + """ + calc_bp + """ + c_count = 0.0 + r_count = 0.0 + for pair in pair_list: + pred_tokens, gold_tokens = pair + c_count += len(pred_tokens) + r_count += len(gold_tokens) + bp = 1 + if c_count < r_count: + bp = math.exp(1 - r_count / c_count) + return bp + + +def calc_cover_rate(pair_list, ngram): + """ + calc_cover_rate + """ + result = [0.0, 0.0] # [cover_count, total_count] + for pair in pair_list: + pred_tokens, gold_tokens = pair + count(pred_tokens, gold_tokens, ngram, result) + cover_rate = result[0] / result[1] + return cover_rate + + +def calc_bleu(pair_list): + """ + calc_bleu + """ + bp = calc_bp(pair_list) + cover_rate1 = calc_cover_rate(pair_list, 1) + cover_rate2 = calc_cover_rate(pair_list, 2) + cover_rate3 = calc_cover_rate(pair_list, 3) + bleu1 = 0 + bleu2 = 0 + bleu3 = 0 + if cover_rate1 > 0: + bleu1 = bp * math.exp(math.log(cover_rate1)) + if cover_rate2 > 0: + bleu2 = bp * math.exp((math.log(cover_rate1) + math.log(cover_rate2)) / 2) + if cover_rate3 > 0: + bleu3 = bp * math.exp((math.log(cover_rate1) + math.log(cover_rate2) + math.log(cover_rate3)) / 3) + return [bleu1, bleu2] + + +def calc_distinct_ngram(pair_list, ngram): + """ + calc_distinct_ngram + """ + ngram_total = 0.0 + ngram_distinct_count = 0.0 + pred_dict = {} + for predict_tokens, _ in pair_list: + get_dict(predict_tokens, ngram, pred_dict) + for key, freq in pred_dict.items(): + ngram_total += freq + ngram_distinct_count += 1 + #if freq == 1: + # ngram_distinct_count += freq + return ngram_distinct_count / ngram_total + + +def calc_distinct(pair_list): + """ + calc_distinct + """ + distinct1 = calc_distinct_ngram(pair_list, 1) + distinct2 = calc_distinct_ngram(pair_list, 2) + return [distinct1, distinct2] + + +def calc_f1(data): + """ + calc_f1 + """ + golden_char_total = 0.0 + pred_char_total = 0.0 + hit_char_total = 0.0 + for response, golden_response in data: + golden_response = "".join(golden_response).decode("utf8") + response = "".join(response).decode("utf8") + #golden_response = "".join(golden_response) + #response = "".join(response) + common = Counter(response) & Counter(golden_response) + hit_char_total += sum(common.values()) + golden_char_total += len(golden_response) + pred_char_total += len(response) + p = hit_char_total / pred_char_total + r = hit_char_total / golden_char_total + f1 = 2 * p * r / (p + r) + return f1 + + +eval_file = sys.argv[1] +sents = [] +for line in open(eval_file): + tk = line.strip().split("\t") + if len(tk) < 2: + continue + pred_tokens = tk[0].strip().split(" ") + gold_tokens = tk[1].strip().split(" ") + sents.append([pred_tokens, gold_tokens]) +# calc f1 +f1 = calc_f1(sents) +# calc bleu +bleu1, bleu2 = calc_bleu(sents) +# calc distinct +distinct1, distinct2 = calc_distinct(sents) + +output_str = "F1: %.2f%%\n" % (f1 * 100) +output_str += "BLEU1: %.3f%%\n" % bleu1 +output_str += "BLEU2: %.3f%%\n" % bleu2 +output_str += "DISTINCT1: %.3f%%\n" % distinct1 +output_str += "DISTINCT2: %.3f%%\n" % distinct2 +sys.stdout.write(output_str) diff --git a/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/topic_materialization.py b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/topic_materialization.py new file mode 100755 index 00000000..90c3b38c --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/generative_paddle/tools/topic_materialization.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: topic_materialization.py +""" + +from __future__ import print_function + +import sys +import json + +reload(sys) +sys.setdefaultencoding('utf8') + + +def topic_materialization(input_file, output_file, topic_file): + """ + topic_materialization + """ + inputs = [line.strip() for line in open(input_file, 'r')] + topics = [line.strip() for line in open(topic_file, 'r')] + + assert len(inputs) == len(topics) + + fout = open(output_file, 'w') + for i, text in enumerate(inputs): + topic_dict = json.loads(topics[i], encoding="utf-8") + topic_list = sorted(topic_dict.items(), key=lambda item: len(item[1]), reverse=True) + for key, value in topic_list: + text = text.replace(key, value) + fout.write(text + "\n") + + fout.close() + + +def main(): + """ + main + """ + topic_materialization(sys.argv[1], + sys.argv[2], + sys.argv[3]) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/images/proactive_conversation_case.png b/PaddleNLP/Research/ACL2019-DuConv/images/proactive_conversation_case.png new file mode 100644 index 0000000000000000000000000000000000000000..9b95e7fcdc9ac17323fd435b9c7765e366558bda GIT binary patch literal 190506 zcmb?@Wn5Hi_ckTn-5p8|9g>nNASftEcSz?9-6g4l^iV1&9g;&gNT)Q6bPnCTn{)o> zoadY;KE3?r!)CK*@B3c+UiZ4zwXQ{^#xrF+Y-(&IBqY2ij}=}ZA>DIELP9RWLUqqF)&M z|7b=J@xkT9b&2fhxEG6zER8FRl>5A;+7>sO@uHl!n_JZr7Qn^!U5x7SE%NgAub>=a zbo5SKM!j0tC#3dW)E+jCgAx?`;QYx~Y{ms!N?l!D>-XyzL0PQdbI=qd)+n^dExuSK+!sTU z=X}{B2zsrq+|u1V^ga-obeC{E6SZZj=e`FzQy3CvnK&^n?u`l+rOpjwyQ&uzCHXoZ z-@dwH#8)Z-KFoDNzU{OR`np=^!eQ@vk^=(?8Ib6)p1$r$=qChh^x8&C$4y5~^`V)g zJ+Fzmqp1Zi#NG+8Arg`#zs@^9u3uF-v1JGBQfKm|H%4p`i4CUk84ZVzzd3b9x8@fx%#2upqCaixr4p zTwEN)Cjb%<-~q1SarJa?GlB3pxU&3ylRxgGVBu=!V&mjy;Fv)m>}rxFCcziKG2`{2EHnJclM#Wi;V^FBBRZ{xnf=K}n;R(Cfp0 zj6gU7DHH1|!~f>oA=CH7xqi2bdp{ zt8BA$kbsAmI}A7MM?OW)RO?my3+6rdzcu(O{&#ByzL7^lOyFCV`O^sp5h zHh4K)?o@-_wH6)?moI_Nu=%7{Q#S5MmUYJ~nWFktDxO*KXzRmNrJ z)pFWKqBGunGX_#(=Ymt=tXz^mUc`MF6qs%3DbLBK`)h?cRp#Oa<0sdU2YQw%7VkmU zUQf81Nb;m2|J4m-6^Su}ct-Exke_`+LwNkONOJF+2FvYCDJv>YXkU{5ac9E7Z>j;{ z;^_&E#v$sVGRDo$xCa}NGzLV5BRO&oZVjvdu*5eGWNFs@<`vL17Xg93+=DS;CeitZ z60i3|&;NS9tTPsbK>x5^F^cV}oCp`waygOz{pQNjJ{(aR>7qYpb?0t@n~DLa{?bgz zVUEbRzNO~1v1%6ox}GdJnk8P*UF!Yt{;0-3GV*#=ly8RdntkK#^MAOZtUFUP7>R*! zI4+`;R;_@ys{sC&I|sgHU0f&;h^WpsOi`STm)HFJYWmX{zaIo*voI1>w+`8M{p(2n zc^sbxIPz?Cg8W}g`meS2$MWHf1}?&uFQJS2*O&8;V_63jVrN&u7x@2cRQY_sMO?TI z9{+IQzwb1)SdZKL`yY${`$+yelhq3BQbl}6(`El_Bm2e9NF*xCvhtS^|6!%Om-qkE zP_ng=HBDCsQJ23ysLzTn_vhSdSRKsV>x`vaV^@djujjmk{;zS9X2{N& z?wSl_`q$Aoz=r2MX6%arhb`93d*(2ZF1}kaD1F#hr2F0Fcyrt}oWT$B{2QVfm7wNu zP-ajql<1b>D zfDILlak8_;OD}Yvtsx#~t{x0NTmw1zUYKKp$sW+kLLtc&TuV6O}>QY(mxj_Fied=G{3VYjMJ4Qqc3BN zB~3Rj{vh3i_g_Mf&~DD)1mMGdQ3`{C$4%hRZ-KQaYl)(WXqP(w^0A%`SK08^OzgT zJqqWhvt=CcS_ZgIcv63Kr*;i0eh&3L`Dt_L0~liWd@U24>3>t(QEtCh=P@@|1Uu+u zJMi0Y!Prf)f;!9GUR7)jWd{R8nDa@Wi+|4Sz?l8@-xiFlJIns4x=byvfk!#Nb(P0x zo;nx^f2#D%W+Ul-7aI=k5k$NBxv@{b*se05L{We|tSPsOstt*Iq%XI3I%8<|PNs5g z1=IxZjITk3Nvhv&!N+e`(J#`sb+$QKV#K_#ztlFT0&4gbhEKN%$%lHbruYSvhtqp? z+n4vSOUo3!y$g(vPhc0D1#`*_zQ0ap9AzfU%#@X32LBpX{@5$DtWeypyF9!p$?N!cNWZvb- z^jVH8SBxU3iaVLmb96WIJx)|%C6-p7t8<@wr%b;wp=--?I%O7D>pCFb6>g>jz=i9J z8Ey~Wl_d+6KIRQj(^a|mQYgi205U4AjH;eP?dPr0ytw+!Zv{TvsennWW}9C9r{(dI zW2hbsI_;!!teG_Q=7DFA6&uX>i~0kxn(A<@oovDPG(%$J(}&qoleRa@OIyi6$oa+? z7!+0GHrZ_1pPER7F(;-JL%o78vCXI5<)_hD2D2?J3mf!5Ur9PomU`68=6o05tkRvs zBz7uCee%Q1V#PbZJ1)$_TnE8`Prh`K>Eo$R&&@xRcLTweg ztX~YmnROkMS~K)PD1x`ky1aIb-yKi+PERaNjl(=$aT;pgjyHYK^WOR1w#uuwt~f#G z(#>R8owX~LT-0=PQEX7}(KwK+9Cy+cRiQJ6-NCK<<+p&%XoT|pus_+c5l_!?y@Gg9 zB#o)n)PB2*9kT|2WqDFpM$SBXRorDZ-M0f0duQ$ih2?K)gskH!AVeXzd$%`(GM`>k zN&2Xe=AiwDn3Ru>#{MjQJuXNCX`_|2lfbzR@Amu^$2EX5oo5`I8g4zlaBy#V?2dF~ zc<}_1`)(_&;b3h?EX(INiW=My?=`JRxll9@YsQQHQPftiD^PjU!R#>UuDo4np z>EkYuV-)o9*8hkzk*8)NiIcvtK3xiDsB62RTb*{Co=@sJp~N?VbTe}<_+9So0)O)0 zm^o+;M4dAL&#fG8J7kj$6Fru%k-ErdX*lbBDhKk|o&7oI)P^UGzi;Thvx32CW_)ZM zQHDY56kue|=h3;)@-AAZW}Cv-q%#^M?;*p(T9FX+{kIL@llXp3#7tGc3{OC14H1~* z!#(vG>!D-9w}Ihiv#G+is~^aDbQAOy3rzB!Wz72D95TodbI8aE{3pZhqP3oSgyWyb zD3d_A^=pRWI7X`-jjDmELXc~0CyRGEX>8HUjz8)-AYKDL?GN7goOI#YnUT%Gjj!cX z4brT35MZG4m~~bL!2}rM8lrUiAfjAFmBa;e-S1fF7LFrYL$p}Am%rvgHwj~?%)%dy zU^$m=d`YYOx!}Ji<<60|#tp5BXmV`2u1dRx$N5P$cmO_60RB2Ip@Z2lRiIsxFcg&_ z<$Yq8iAOLqRFHJN4Lcktep&D;cAqpDg8*NVEw97)Be(u<8(%!fotJv=HB|2fHQ%^; z!aKLRQE4yNa|z%D?~4N6P+KU^?0}uv9XF_|4PPRmyW~GEr((4Ap0`C+x*MIrk8s&= zA4zI8Ghr}nQy16dNbJE5Vb$Cgqp0L-*X*7*Aw>bX}&-3Qf~GR95caA!gvFTmT>T6ksmx zZ}P27*m_nO?k}netd`KG9jj6ZzVELj-G&h713={|v{Y98r;(~^jPBM%(Y5g}(K$$N zngqDmsH2fGOL7W$eXEql_VqheryGTE0)o2;KaJDP1Y)Av950|C|DY)p93wK_&4gZE zu^(;L87<7@{oOpdvpmdD75t;a^rPhc{=1!8mi}(08|K>21U?*k$!&r z7sC7l{VHT(lgM_r&qJ<{KdsqGhYNZhm}lZW5_Z@st}j-98^#~K)>EZ=HM?M+RgQpn zUD|&GIQ>cY5HEH4h6YjDwHg6F5i~_ZU?Z{YlW% z;Ai+B=g3LIn81WEbM|9?X2+W*?OQk4nk+B{B-Fck!~RF+2K^ypZox6ihZdE*oxm{qb zFIf(xZ>;vGC7ocHACGIFmbu@7!Y&^^U6M9qn-D|DZtYqtr`kU}#%9i*+G-q_U@OSc730-4q#ply#C@Ipf62={)>V1PwlhDWNz zas=vVMDGGm;ftV*O$m}!{#TpjiHazW&HgF9FxVT&>yz} z=!l{OIpl~BZ-oSXsk9CM@K|OkghX|}VJd2vX(3(QmA1^YZP9`$V?&X`fYjiqtLpgu zqoG5*hN(&$O(XPWMB(d-+1xnk&7`lQ0Q9?!-P5$Q^b5b>xLeBNm}6y5XsA6EQ*~z zhp7GZQlq!S4}S>Gk0w3pM8q4|d`|Se(me=i`9al;3!1X-4gprlzTm57Q5+zHZ+dFtv7r559|G^FaOrGOI0ZN41sq*RH*YBS(mf#&R!8ye;^`gJ>NyWr02H`>m*Ay~|F1*-B@b}5(h0rER;xc!cHj1az)42V zic~7!Jky#^rfnSn6OHS(X9(MVtUnx(FkpU{@5!N~3@|IDT4-;^zpda)?#osK7lOu5 zrXT;?IJ4dm#YgBpTPyGjM&GR*ea4m7@G^5QnpS+I;pSp%d@W#vwg-SC*37$f#maHl zqlGWu-;FeZb$=+BppIQFwZ`xIB2oXf+6!`C{dIuY*h=ojR>`vH`*_D+g6y(6)&Bm> ztbBl;CMPSwrU<;kw5zl6W^iF?n(bu+QqNH=A0U|@r*HYy^I;CrMYra~;3WH9X5v7y znZfU@#TVH3cNx@}A3ah2zH&t-PF3J83;G49& zB)BkJxuoqc#_$dzeN)Nu7%!@O!~1Wk)SvH}5oJ6Rc+;EyvF;)Nc=Esg`tpts=kz(y za{up3D{xaCc%R2IT@U5{$3>iX1U;Fw=eN2)CiJf>`1fPkV-8@_7nx>s+5h7rGJyN4 z#?F*{Y4G>=@ULq$>w*MW1U-X@beUT>2A};vv$xj`*6-2;t)8XWs(Ip2@N&C~bN=xS za**Nn7xm2DY;Q@z_0Y4c4fX!@PI(rK1G?@pZNo9S7JxbnME7fH zC}bH+yWka1^{`UWbsa#SU#wKwPuK7o)YTkM=naAv04jxT)2%H=wC)a*D?G#k7$o=V z1;2XR;T*+snD6P_WKr#EszVnY*lkeqP_@?sNRAWrzM9Ba12?$5o-p$B)OM_0xw-;i z^Va6XSNJP{CP=Wa)DXl{O%v(wDCu&u>By?+ZvT827a07n$|yUM>V6pcop(q9T#>`k z`tUCBygu7YfGrrAbLr#pT;Z|&0z{ZDsf+d8j#|BxfBB87crZGi%Wt{-ZVh~v#u4`1ELIQW->A41MZjvF?|4`IfDiyzntj;n zyT7Eo3#8NTxk@oENrqluz1LeXKCoTe%vVO>X?{_4AC$f<*9- z?e5m^JzI4IqS)nC6lfMGglwM9R2fKA4|mdBo&yj6iK9j1P-7zC|1z^Bm}*zs(DUv^7iJmNqXTfaRdW#d*!Gx7UwL6TG$1E_i6I&jth+q z5@(CSa{%Q~u`4+Gv^WJI=mZo)RFgj1Tv}fM-lpxa9iJ5t!cj%XE=w`99rijqq&Af!d~+|RoE3E%3LSvoCroVT!s}XEc&%m6A_1M zs2Wx1X&NW}q6^#4WUh}&b;dS;jDE*s&Ru&F$W%`c`as^pG-dHh#cc#Ku&hg{m~+ky zuvq2*QV3hu!5tKBwS0$s2@U~j)hTJlap)d}{-U*8{9na^Kb(>@3~>BbCvM0Cv#wZo zDX$|Z_cyoRZy$PWuk+H1@zs_9wF|E@;gonwz;RG%bL__Gg$L&se&FnXpEPs@SXu|S zm%Ud{xr!3Xf$iLEvdP~+ll)j~#b^(X_`>pEO_$PR(7DBL{j9L+y&3)d*}S_nAd9xRW%qJ#Z!6-I z!SLIp0XTyqih^4wzFZi(k?D8wwW;LAE3M{Mrf~YrjuA`t`4QRC8OiNifG{Q&&NgAC zTz-?S*mrh5Fm6n;i2?^KDa}o8|29b^Mk!(sNOV6*0IbxtPfc;HG>(n;cA4_QJg0f> zcrCZ&ERnM(?;RJ~BhQ9QI7mmGVeQ0C06&fP8j=qUE1qxgt|cj==nKT4U-vF>!b=>t zKFfN9%OuPE#00Q`Zj0EPvkych$kJRZb99s0E>G;968*itavupO@ZJm1iDo7QeR+001@o?QZO_D2 z0=tE16?(L*h!(1g#;gKdOC0R193YTtiVeWk@p#pB0K^oYv$Ponn9~)hn0;1-v9_F* zyt!`TQLNUKgIB9ZitMizm)!44x*p^r%#=_|Oy@jZ?HgdyAXW-{>_JU+sc|07>f(BU znI7E@kEepP^2At37-3TCheL9mr4lh10- zjkN?aSA1+LuXR3?lIlGMH;xKVwTp0VE-$tO(?M%~%F!&I>*&Vq>nHM(!}TwDetA)U zILjUWn(Pl*RJyf>F#f{GeuimJ94*!f={b~SE_UX(2kxq-cLn50djn-k_vL>006@v+ zFoE6qbJ$m@m)1HgpN0W!y|ey=gC7@azwd3^}k&Mm|e5Imz4mx zxr|%Bqm72IA2RICs7ja`bHH*+f#jtH=F9OqdX(RfFI9CW#I4V~BjG@jQdo80cy%_oWY2hdkHpG|jQS|N4O8!&;G9TvXO8Xg++5(`Q zH43si_}JcuSdIdnbOrLoDVy1#8P#!D#5uz|YukAI4h&9bcMSb^+`EL|Hb;f;7fr|l zshW`|sK#1!%bcy^2O|oR-eEQyJ^Xd~M;%3&el#CL3*p6X)2$z9(P?Mono541ZGaPx zYSs~A62^yD*aRTk2JO8&-hV$fkH%kQPyHkq5?f7=B1#{br0)_*ar1znq?vD>WDTfV zy*fOlH;jJXHgm2-_rGothJcbZMNuoW1h(e z;~~m&Ek+-=EV{LK$9e^>)qT0|*d5P=kDyjJZYC8er-m?P4+ood zl6;qpzI=}!&}-e^y+VcDCyy`Qe!6*INorJhw=@&jk{0%2`PdL!IhDtlcnl`JCp$Ba zn^8-mfwp?={{*O^cc?dB2uGis`B}BY9IPgc(IX8@Lfg_xxckB^`=suqkWS~nd@O5#84^mlZ`{yUQni8qZ|p5{CE!AQ70f z-)q@P4<#8Z@40W9mz5&~6m+_t>g`$oG?CT|?(g-WeRLo1+|@ucoo0$JpSIAX+c-QT z%_)LrEeMK9t1S}s);*%asdf_Gw4HM5-4^N6RS3zmeVg-N2r8D_nm8iU+#Dm3&JbOb zW_fBp!o&UKI3w`8Wt@jJ%cJC#L(HT~ZCK0_riawFh0jBzK7kH^Iv;|o1|#>Cu8B;Y zy&IH|SEmxetqH$J{{vu~;*Z0qcJXDFACpvS9}eR0dKZ@r4Jmj+@JVz-*N&q>RyF9y zG-v1rXy)iwI7`0WT>scH@*9C<>(zm)%H;lOf5o;3iR%CWl|h({L1>QCBU=6d#x?Yj zqs8yc;vST_g;8jGOqqnorwb@9UdLg%L(~z&a{!HA$^5EUX{9v4iaSRG*5@g{{jkJ9 zLk4yR_1+Z+#S&BJ4uvH#%h8)M&^Ox_egBwdf9{DoTfiiIlv%m~ zP_-pR2%q8ZKX6&icngUe4y`xKd&Uc40I4)_vcd!Jsg?a&#HN^Fb(d?OL0l*TS$ixiA|jfR6oR{CJm$!wWD-h%N@7)rc~5b za?&Bn3RO)#dN!qzGDFbW*oZ0~8OVqPYRJiHw1Pe-(N~+Q+M^Fhjhb z)S%evh4FE1$8Bhp)e!ZBS+H$_A^+7tTi5~S&Lc5gYJHneSWTBlt&6F@=ATf|NE%!x z{A0h+I0gh2MW5CPv+qZIgQOvlCqJuDOb)|#iv=8K_xZI`H%ap19EOH<(lMjl2-ttf zO&Qe*5Z#+<%!nj$+rpoB*f{v8OwDiuk!IG>@?beaq0fNyC`w`AymSodnkkZyLZLNG z6HPVtcpLh<+Z=MwLtqK?20)b>Zv1PK&@i2@s7@FZJQW5Opl`=4e`GujfyXr5ulW`kxPc35g6N z@5Y#o?cVrf@rcga1GGUSC47LiXM|(O>V5UV{-~N0Njm@+JKD+HDMg%^ z7;i(~+HYt2mZ`&@*uXAU(;Q!TsEB11i_}o!JuY23@ApV44o-+sscl9KCiAxud4(t; zQ+wA?!1-MdiEb-)Qafp=%P1e0C3YY;4tPC6Be>L$p@0mVLFSmpOoiSH5Yph*60qw{ z5$`4vb_Pq{RbEj$L$?xz{!uvpJ==c|HIw8b(#Z|pUc^(OZF$Z5F8gEDdwA6CS_amWq%jod z9;fWTy2IH?o#~YJX54og4ayR-nMTGXG^x8tit9gr1;b{2XzX%mf%n)>3lw%Uea}9g z-HhfwPK>ul+11t~}+m#c`%P(1V{8c(#L5x9Q${{(GM#k*O%1s+F(71;G`s*B+536wKD%?@IOKVk zA4};&_Zwd_9OtpxB!0i#G61#BoramvL!J3k-GJ98yTZ|Y$;rRs3643X=JePBYO0&y zmc~BQBbUW%(fPKo+!nk$Rndygd)`7J6C)o#@BAO3lEq1QcG{QCIaY-}jsPmr2!<3N zYjn8$d07{|d)HLN_C|DIC|{jV7hc)h7LO6C9B$^_@ri!u*|!$ULFkTEZf+BNBSJ$- z)IxzAmb~2AH)gbY@1>T!P^nCv`(asB^-I`{L)i%f8jb>v?Vws*p?O{VwenLZu6%)QK=;Dr?dswy%gT$@wWjr;E?Mua76|TY@n- zL_1JZF1a&gRjJG3iNbP%e_X!_{;*@u?Cm)3>Pss=KkQSTalMrvnS#?6sardHinr?1qA)8_L&jW?6z-pFY zCpw>@z!T>w0nY)P4zaotxgRVM%-F>KX=`{fkLO3nSEFo%^vW2ZIFj=F0eE5Yvks#k z#qd{j4LU8ZK~OAAim?w5KPFA}3%gf&&L8@)J`-pv`b>1v#-C`ZJKY)OX5reT@3rxH zULwhg_fB6?1Bgl33_LyqB*CvI0gy8o22YH$QP^c!@)eFx_%XXbqrsc`!v$y@So3C>4-)$!V62AR zk(FF9!Et{VWZ~Ck*rDDJr9q#~9Le9Q%fxViyXbZ0eeotp6hDZF9tc_AKYKb&IG@ge zGkL(Ss4T4xQ`WOJ2A60m7>gtGQhbuhUo`He?)PlW&k$ zC2=zapRJf6`^bAi31Y=$X)=h`70n4MQk30L_vDjVD zK`&rgk>H+&fKcR#*1qE)Hhy*ad{j9NK<3K%+^E==T85lke-VG(H-JGu0v<&V&d1R_ zLjK+Vd_w-D$GOoyX9%$p|QEPZm zP>Cn94Wh-E0dfz>ZZlqq5>!w!xW+%nup)uvd#)XlBJ~q%zBePEzPbzUQ5o#OGY+D0 z=YIU`T4$RZYCG;cjD$2mngt`wiY2$38D@w<-GQhWUXF`E?mmih!~;l+x26}JjFb#bld7$79sYOln$od zCj24p%qS=R1#3bTf?0FeXaCSH7a4`XjQZPMP0oU4e^~`omt^E6ZqLN<+KjuZOxP<= z&`TB(+b>Ay49Xyy{wmZuqkn1e**Ws`X5HJXt~}QUx3Q<*Nt)Tgt^Od@im=bNxORG6 zuNb=bvaKR|0S(>mlQv~jU&L~>ge+#RzaIy+PF(PXH13Ohjy>-Uzd^!a{`oj+9Tt<> zv+Fo0Myu6u@?*H~Y(7a@XD&1E@;<${n|pz%;hb=kk4y}m#K64ct+Piq?MX{L;a=)e z2DhQ~5_k&COO$lVN^2`8&k{3y#GcKh{|9Qi^+80+e`X2LX5^>k*DdoDXByy;x!EWL z?-{c99&?txozOF6Jy)^9q=6i@H+bP!f3!qjd13_%K#Wj|wo2`zP#@FBu00qN09o7w z&l+YC@dg<`&p&{#git7ln1--OCJ}*xm`s_?9gX>kmfBSA8yKFOLF+orQo`(21Ql7A z?vWBG4Im$5IqQ4Y2FvroR3TXH5a(Yj$#M8-xFMbLsu3M50ZDB{9==XR8aFl0a9UHz zPT_zgi->w`+hxu?@AHP}WR069`#h`222gsTuYeG_;ywqB*calQwvY#X%A-5wQISkA3ADoST;LA_ zD2vPzkh35Wbudy?HUH_{5R?2h143UuIDEl*o8i)T9Q`tgf9YNg!~PFn!1X?y3P44d z()Gq zRD_a>rsIHUS+UBfaa$MG<+7v;58*P~jBv-WlM&-Ivn5Ft3XL@b>(9Dxu5p6vvtQC2 z-VBmT8x9;TJxLX^UnY(sf#MVf8#vA<&YJNpnvus@D`>+!gI9XxSvrCVq}NwxJeMwV z%7S!$Iqlg6vC_Y5C`7Q(n1z^0zPK&74Yy)~&OvaJ!VyG0U6QmP%Pzaqq*$UXBq=;s zoid%S;W2$>yOo+0W(%`2X`b{4wCy-aOq^%y8aH<%|PdfPSQE!b4j zQrzU1>`yLQ4c=ntFAJJliKyA$0yGE5duy;ug_y0Nzy}pnO$8C%xY2#Ji`7w%_+Ib# zX9vfu1KIzN+ z&)z;OgJRpauDuw%40=L737s|2#kxSOeMH=pwYb}p>rO95a|xsKUy&bEY#Ny9_Em$vi779+T9gSpXIW-XZX??4Gn4&L)}wiOdPEq( zDWNWVBOWtm3`Mp!S`0<3| zb7-l^1Fi#|EdC9kath$M4~N}F+`=hHgLWa0piM$rqivzK$zfFUU>B;U5&}uS7B45f zW2V?{^+9w$TbrY$>j3X{T9SO14r%pjkROYc9%h*;iea67j*Zack>nrOwBOyBc?Tsn zdn-gCFAlDBzBm{t=ZbfQ@~fM%4boDF!<7i=ye@{aqN=6$R6<@FFRHE9P7>`oEn6o$Y>~Ig9fANAB=4dR-m)S68BO2PLI^q(W{T3shLQosN&S+Sx*AyOdC!XCcp6 zdR>c`nmt)TsO}?t48pq@&>bOC*MaqW)xSPWOEB)!kf!6cdcgXYa0B?#`_Fc^jC0)i zBtyhGt_FtMRDX{p8opTjQfejmey3-#vIR@^oNwH7aZ)J)zbQ3GiMwx-HcZ2k5oOId zlpw!o_5o!)&}-OZmS<>XK@a4@D7d-l21|OaBOW+|0?Q{zK!L`Hq>Xs!AnBJC6TZa~ zKtDqT?ouZ6|IANc6uc zdX{lpS*m(;W$kAyhgRdfMuee|LkA7Wx?Q$(1@tDjkOIxW#d=6C8Y*P--ci+*iog(7 zv2TuJs@Yxg6;q2H1%@%KVmDFf_@0^9Ff60U3e}+FT$agvk|j=Z**fGw(Xq_6n-Tf) z<_M)b?=dP$wUu9Z#Q;cZ3kyA^6wR4YWYvgAfnLrF4W?s!=tXNd#_st<;Gd}YeLkNaWhjqgjPGHRLX?)?g(hJ$Ja8jY zqLi!D!EXtY>EpN~=mIxYk<^_fhao2jyH71qHY0}<(72aNuXXO5$jsI9s=zxo*kA0F z%3Fcn!V_tP<+8gKfQ%56{$jXp`Ss&qvEW@|W-QQX{$r~CGzTa;VQV5ICa7qbGC1Jr zI<0A`74Z~ij6$`~@;LD{H$yJ+T9go~wXc0PrgbvY_JAe3>EU)H&0gZ@7c~`jlKNY~ zwGZ5*y_VSpxl@*;g!Nb~3B@Z`fIaE~L(I3MpuS3JUGz0^YA7DSgW6vqwf6686M6jL z?-IKac#U&-W+D4>z}<;c_Uy|kD*Z@`+)U^JrYI3X?$HO&Bh!@PvQqoH2lM;56Et-7 z%m&hTX%QeF8N&tCHMX6e6VJ&5kmMZfNjM<2t8A$Rt|PDr+j+}ROV0xW9@VMdDHgs- zAVY(Nba@;HVT`F2y?^v&HR9#|Dl$@w_(V9d^o*Ue*FDZwW7-R)Py*6|k4qg4~ zm+fvV!^lC8er<104~_7`EO2gvCj|N+dRetp8YhP2IEOz&!;02jON*rdS&2(^g!?do z1)Q4XV={9AA;)pmOFFfO5<+)jm4-K4#I!YhdFm4zD@oUn1peKZ6jI z0xcg_PDYs~Ju(A=^9v#6SxZQHxUGgx#p^50$waV8l3)?aNmBSTryEzS8au%@IUAuj zM~;I+BXYFIgj~s$J~6&<#@6Q6KV;dzH&{M>lojileh)bbQes$GS~6bkR0uH=`c~4Lg9*`t3T()Qbyr0$oT~At$cuvqoCyvCrqCZ z4cTN`3nm3f9ClSbo<`*913Yq=Ymq(5W;lb-I>s_9ft)^}Ga`qr9XaQ|M+aHbW$^{T zRM;3Y^Fnh(XONn3tDeHP&H}!*f4S+DYAW%5iog3XQD3tm+W! zA7^2@xMEdusM{xQ;TqfPzA$j);h%=6_=ZSvof#F{NQmw8f2c zLWkpu*o?p(rb@SbRSy8Y!~~s_tRuq(dnVsQ6Q};p7)?d}0i22Z#2(z+pqYeymbNfF zS6n4+YV~6~i#Hj+No-`t+4uKoG{>d!#|_bXLjz8^D<+%_NrS=nfsO*;}^ z&c-W|fNcQ9+sT$)!?MU@*5V*EF*4I7Mlzy5CAlw;CAdjbT;JyL7$!e&;kRpN4F8lz zgUR))#Y`GAY`ErggagET-X@q#Ngg|7Cr7j09v7qmdm0&))<>w1rAGK2sO;+{N808* z@ssAW3al$Gf7tGQtR zT?-drL04dVC^*NOSR#W4D9p49zKr}+W;LGK>^_O-n!yvx1N~C}8m_R==&Ra2*X{k9 zd0K#2kUR#NuDw`v9x@kI^YcEDwX1<=8D#hVk%>PH{t z2sFv)O}2zsMo*0cjU0KI9Aj9}Lmi;jK4%h6w~E2kJ}MfLj^Hx84YH1kh~kSGdb{?9 zy)8Ep<}APHcYEFQpa&ATTVrN*8jSU|{gb^3zfmTjY+`^iv%%lr3z@88?ETin{i6fu zXlSSzSTz0_Ew<6J|4b3{7HEnyINhDQh7g+=)IET?&S|vq zyP?GHQicRnNyV zKig#$dL(|(oB|d6fn?!`n&_f{hNF=_!^KV(@i(rkR~v1Lgy@VAPUXq2wYB7M1kkr) z0qxh*`LWm!6MHPir=TP0kUO7aqK9q)2RXRybYYT|0o{&l+AO%Enmidysl*RWvORN% z7bpql7$JFm>eMK%58Lz}hOl97`ClJv%S2%abdh~;7D_z{?I>7rTp!9FHkY{tgcww( zNwYwWZR6}lHafh3PBN|y%E zzQ|a7awoA8yE@IpFB9(8a{>rCWY+jmmKujDI)7|S9<~YOmE~daETGfAb^z2@jfM4Q z@!$289^aG07+R4@3w1afHnW?q{OS$#>AeuSTnr|lHEceg0`kN;zt!`%+T8A#U8<7H z86ML;L~t@^aR&LC&$cp%*SRRX%vZCuuA}%DVby{t0?;PRoDA?(OV>%`ZrxV-E+Bbo zQl;?=LLO*kv_hav0Qw5Zdm|ByMdC20jDb-x2*D_P7bxpJ`MC=$u_{QPb`8Ch7wagM z^^RxMXq1(xQg?(C9fzQzCFNMOV+)z;dcOtGJb!A~28g_R6JKuquD^dn3Ftjm+*)%H z^}RMp(`sZ)Lq#*v#WthtJ&yvvJ^ubu4FEydtN~Wj%wOyQIuwNfLiHmbly4v9GbW2{y1l74qJX`~g%fg%fQIskdXe4{ z5=`}u43V{uT1&wXD=hmDo~WtT@PrV>BELJ~I^_I<<1ciCvP8_Tn!Jed#GJ7j&{c`B z`6&9_3Q^Ph3_Xes>|N$J-{7eRyg&@?98gV2TWBiiqgkOX^4*^S1Z2xz$GtbI93J)$ zS>_QyLsDw_=0;xT$r3jul0XKZ3IuhTNb{6VRiTrx6;agzt=_Vb!toAl!z-j*=XHRf zlL(j{t@A@)>i}zYQsO3v=!RKuuLW4(0VAi<4E7jes z=_u>;U=3ODZH{a76clXUfhXBoLhI^{FAkM4E2r0!_E7m-KK%dD-p6P!DJ0TDG8$j%Sy@YobJP@ZYy;PA~GXF zfySU%`O@4znLbPx)xI{~+F!(?Z^rI7akXM_$IJl=Om;}slgO36MaTBW7~S<)2DPZc z1wbpe`-w!|jV6sx+Gd0`9K(7xLOe<=a?+XW(6nJ9#BuqwGX_!x$*h{U&Ls}cf0txv z9oFY<%fmQP$GStJ{6GGQgx$wzM$F1-b` z(=z}OIuR8ULLzI2M^5JSq)kthS2Y}FlFIAh#LgrbvXJ$hkq>Ky4{r$%(>9PvA6!A% z0Pl=D?OS4iwnAo@f4Qic;JSg65qm(hp8(IVrqxti808NEQ@-G;6|<75?|7&BCv)Hh z@c}Cz*#`X?n}*+U(|?gFCvYL7!5Y~E5Zcr%m93RFqduw{lFkUFXqx%#D>h=Ry&{?k zf?CsvEq^cnYCysj{wYVDpj-phq_h|iuZ+dei1OAB0$Ri#DL_?fX#7iu5FNoB+xeZ2 z-!x=!p}?x-mq(gic^Qozis%x4PS_ehiddNo;)2$!flRJD1waX-a`$}JrR3$;1JWwVYLi*VMlAz?K5pvftWhN5$1<{1}>lbeSlV< z2hiGKcN1x8$m_@IqB~<>KgdktLc>JugvQWOv-x$X3~Tg$mJxcHs7nmg?@{fP(4kq& zDHfJ>ZZ;Av0v+uqOVq~E@hY??=(0Gdb&97=DHWIqbsCy7It$Wh-`S%v@(K z&{EVq3htZn5z}wzW{e=omqgNKUgQE?A%Vu8nm8n`6_LTX#bkDhhc!|w37YTcVV_vc z!^K*d7&PU$w*g&V&!-F?KfC=2wux;ZfZGI_T>rZ8u>!=~n^H2t)jTr>b|yUYnOBW^ zO)D3xcn$CMJXaE7lrdI9(1P{}Jk8i}LZo(~&H~E8c{e4oh&}%^>A7#_)XQ%gJJ0>$ zF`~1840l_ql;}8^X&nkQZX5nu^&ylqIZ7@omJjdunZllM1Qg9ZfuV?awyX9OtBj3Z zL4rFVU#04th=^yGe9UM1&@pGIx`M|)kvN$0AE*8d0GS6DRq{I;)3=s>hMJPt!t zR_ROD2CVo5JW<2s{k_`0Zj^ES> z!X!VQ&mF%CHcn&Gd9={(pV33Fj2o07)JZnALmS! zU5EG$q;YwRP4$NV-P!fXc$bTzsE;>$xV^Ajk)k}I)aCu~z>6TO#`9N9@PDgf@C8qp zSa#>EH(RWC|MjTLFfU_(E$R7SWzve(lnMkc@yws6r~B|NYqeG5Q5UhZ&kK8ML>;G} z{G~$}6vWgY0{-Vd9YMJrn=CNvEY$z+i|D`ooY+1U*dQ=K@M-4le~QTe8wU6c3o%5) z6}tTYy%r-Hs1=N}3#|S>qI>^N97tvLz(OO$WuIZ0*1!GvnuZ8VBHZg$ezkvFLNfUQ zgsJvHvGZ^Xd3EqF*lzcL4Ha8o%1_Kpy+A^dfGERE184I_Tq2V8I~Y`g5lLcH^mL zBU(3|y}dZ_2vo+4hl2gXp3SjqT|gbJzKY(y&7E*+0q5A*<4A({x%3M zsVzH}ZsYU)$@0?~&Jb3^q}A0@2z=IT+s9N;JCrV{ew4$QxqSfq4fmeRnN4~7c{Ia# zdbu{UClE>tPzaN9zLyix{3~sm0aE{s$8XP|pf=990oFwrOQg@8(PiH0U}64j|9Af$ zu%IloCoaNE85W6ySkt(gisRii`8J<3Ck`VU`1txmE6@J{PEcJ&Je&~NO#7jm+3caV zT`{WgJdgQ@YK=0vGf0-!!xDTj&wcvGQF;1f#_Ur>?4i{JR+fx{XKbalE1%!gU9W37 zBUXH__$;7;C3+g=IQ<}W(5?rfl_z~@ic*UyhA>D$Pqrs`OatXb)A|^hlCB@TNvWW? zx|kz8C-t?%1W_%ji$%Fo-}fei!D7*%Bw@kTz<>Ua3+)O;$52<_ZFjL9t#V`rM}az) z9E46q5PyQX+QuKP=g&!fW~vlD1M?N1&+0v!ocMir`9!{fM)OTaGpL>2oErmV8N+`o zIAFz_xUCN=KUz**AcRF&IE~D7f_eG&09B?rTg3oBCW%|^Jv_aaaA>0>{{J~{@Wu7d zcc)=&5$nvR3#POENTFl7mO&t3#LVLAErZ9!iEaXnb^vnF-E~0n;;$1C@k43c%y7GA z$Mm;Ekx7$oYz!VG-38vdkFZP$C~C5>L~S4LE`r|dQ=iPtg7#=W1SE&XV9~Pso5hFg ziV&aVp+hkWZkMQGycGe?U_8fsWdM(`u;(Iri8&t-f&sPc8MX8RxAP~6X@+CIXbH#; zb7(q=aDk(?3Kr*eZyEyFllODI{9gB4to46rS|VCYN2ifD_+{t$*#OhT>=ha9{gLZt z`X(md(xV`=2>|?1r<&Zq#oJ?wb_^TEQR27Mij2*bOu-We43?;`duI=X~F zc4&Ul>tXN2-|M4+>BG36Xatj*q_}y+i&>t;Vg3{wjx1CLUojpK+4ah}KgoQk1*D}^y2FjY`T%w=-f2#Um%7G(fmNtxrtYmcIyypD=wniRS5 zUj+u=Or7{|VaI$b>hCb2(ul5--Ij`Wq{w;4-X zLg!OP}iVZDrrygX<@m<3{sp-A|s@D#{ln zSSv(o1Uoku2L~X2 z4ax1BL=ArCh# z4~j`yQAsas^x`4Y8HxlgLsr?#+e*1qW^s4qTtLj=?i!HUuFoLBbe4}Q{ZYU4 z-a>*Q6M)nK`?~Tn%=HYCP^!*WuHWjr_Pu3S&~lSiH>Mv#K?luMVpYi$nLn8I_kEwz zX#$;kWYBAD!lw=?$;*`{G`ALGkfKl?IYT{y;2{1ouu;6h$saaa2_V8kR2;AQr1x}4N)*r(H; zFhgx4zD9Opo}IM-5)3Y+qpNT&k%iUFFqEZQuEdXNcIKydFLx!Pg}M)8FPPhd2P22%(zU18d_Hzgn@?|&`SPL1TU(Y8MYfJ8w6K;P52K$FphO8&(@*4ON zK%j`qw+CJiCuik>RGD2>?M>PRPF7wI{(C2`3m6naQ@%nzsc;$Y4{WS^RezY3|*zjIKD)j|d_A>q-*1UL|rVR8W8+{tncvsQ+X z-Cpls!=b-dH}{CUl5>=~sN{JfQ`lV23yoKvvw*bJpZs$Lx}=cccz?Co04&dQ$+l6@ z(BY+#^T|fnjhE-8rh9i-LQ4@K#Q%N*#FVC%gl-rTiOY~98^;E3Mx@X)DeBK5T+?Bw zEF+z`CtE9KB^7;rOjkrY=DSnnLmgt75e$?Rjv6AfY&CQC8+FcaM|VK>qk3QVgpoVD zFERQ+ROp&-L)(UYABugRIvrHhe#1IOW#Dold^5nc56+F#tD~fx`>8-u>p!sUl&rHW z9?PA+w2KUd^gK7^37%@lO= zqwD4HD?o;-Gat*pdc$sf$kEv6Cv1?$w_%*9%gfNN6DFAq<DNCtJ&QzZ7+IgBk3{&M$UvN-&MnLxQg^j zWlg~f=hU*ke-_S95wRhpQ~KP19QWMO!}sfSz7lE69v6gwW^XmPLy=y&z7KTKE-#?c5k&ddzHE2EmO5;Zo4BgF0>5F9SH`GTYS1xI+x zIjT%;@mCci*~U z^oRjk-sGkTjMtL@p0&owh4PLFzQt8Y@WlPNO-Og_N-``@(H+}hXX4w=l*)L@3!CV$ zba`hlM1zwrVJ3DcX31?#{$X_y&gP|cs-8)KF5fqVNn$vamxH2QK+$z0gkn!7x*2av zmgM^J6Ls{J4!T2e0Xn*Pm<0};S7CAWHG^I6gL@z2hO5B!AEUXiFLPoro{@>J-lEGd*1{@TV4&iXfT%ScYB5Goz17sI$XtuJP4q(yr{+S7Qoq zkq~U^B-f=IrUx@_iQHk6Badi8A1+qw(W$3~gBv%EIUMU6Xm|F5Jn9UEVt&e?WsK>6 zNkF$zuWT>OCei9|mrc!5Us@Kyv{LEHb`wQA@^LU!j008^BE5-)9Z5iN@GafwWQS3oZ}T- zc12V!#laUhVNY(OSyK1Wv6uc?kX3LhbYQ)OkmZ;n#AUe}fX=#Q=`+~i)yrTX5Z1ct z3pC^9rtjTh#H2BI+>4yAct5kmtqVEsR7d`Zq81T(&S7=rDFO%>a|jzRBbQX>-l9oH z&#jKm;lYGr8x(nhm~JRvrrij4?&eGXVP8ghgaKHZ(L`g;=pGyDCzoEF_Dc+V5O2}z zV;%RfAd=T=oxBmIU472chp|1j%pJPS&Cc$LRrtsiC_#j&ro~)Th*&wSJ>oTp|3_7< z%7lW*R&D7QnODf@`%L$M7|Q%HdI3NUnyA7UG{-%wtb1RwHU6n*Zy6HUxrD31W+3K6 zDSxH`+JHMt>~Ki|U;oZg$J7*;z-32S!aud0((8Y&fBnv=SRwGIL|W4_j(` zRsMMt)Z0NbfKAh8$S0T~iL&6b9%%ZG+BxyyB*MeGazsW15pi+*qUQNm#=ZiBm1b{kt&YDhb3VVnB3&qxz2WUqS&O2dYqY#NLALT=6wQ^s{sh z8vcb`6cQI&q?0!2-Z)WsM~+UJesj*O18chjmL2Xgs zWr!U$$8os_QE$dCI4;v6Zmg+;yX%h+DD+}{A=nLgn8DN<`fZ*l0p82;76{0$FB8_? zs|!-O+ZbubPr6YKh}v9(<)PPeb>A|+Z;ei29wDlzN*S+An*4YS`Gp%pMRcUYAt8a7 zgZYx$d8MP9tUL?{!jDD9b|Y|7#N|=@sJ@SuggVQRjB>s(Cy-C^j$kU`EmHYpGd<4lg9Dy~aB7vq5iG4dK-9QI1q9?U9`qHI~f1Q0IW$0nZm}Gy> zIU~1W9$`2C3Xx3C&mI|=!l$Zi%s(g_>+0BWs_WibmT`PfuGG0e%vj{5O6nxP@5rD; zyA>ukY&z#1c4$099c$&H_1DnQHxOk%Mn(%q9I<_pVchXth|4v(_TOc5y-qeBOVs&%xF}{V4?MFE_QM7y>-YU6D_177Dj2 z@|C#CUl`2CDHg5HhTU;WG`em>ib(AYL5h^=Y5?uUNc~x$XRdRoc1oF#FD``c(#rBM zGp=#uu}s7!pHh}ml1nnXct@fEyg{44OlDWl6y!amNZX|iab4;($=+)C=x0Zx>z$(! z#Wznc5LG{2ObSyTZvKWaN9-Y}t5rEfmWr|TR7pV)3lHxb-iGv_ouT0KFlEZH6LY&z zJjfG=p2lvo=pW#Z^1GLB(5dlC$6S%^w_elGSIs-G~nFL`k!THpzJ{liu}S`_c4~9 zba6G z&>D|Fh;;QAN&C(xi8K#NSafDP22BuXO`zzITUF6z{$*_Xk6xPf2`@`gWI@9v$_g~S z&1lal>^h7(EsCCz&ijRInHy@gVj{39%E)e(a?3c|t(ci`*57y8~yx0oK;R%02x0CPRcu_UAZk3&TxZOPd2 z@g608zwk3wAp~3}F4G%ES7bIN7Oa!}eFYRpLjDKA$DBB`K?}i~X#I6W8(Nfoy}T?^ z#ENbi`i(Om3iDoLMZOtwo;$8%aM|cW)?NG5<(7ucJc5mJbc|L9MSo|(eaul@hpf@% zM;Lhwhk%re?((+hZsVlU$m=u(rM&25-Yb8HZuW#GZqQMAVE58Hu;|6n%rc8GAoZpp zv;-V;ym;2uAa$FRS8f|YbAYkwNV7Nhjz@1i;Q`gho)MY!Nk z2K61AIWuH=uEVa&2JV>ITM82WYJ!3u<=Y>o32&|19*rZ*GVS6To zbHe*!*$*Trzw|PlOjk3|E=G<_RT<;h5>B7F95k)&9W2BNyw}Q7bhZ)g-3Nod%Z*)j_`|- ztFlPw7~^UbL%Mf=z59Mht#`kZPJdm=n`8V@6#0sF%ybVw@@(!eyiS(eR1PDK#_7YG zuH7OE&zCBc{(8TB_sT3MuV6q?FO@cjlT4P=cZi>5siZsBWy6t??Yu8z|H(9YB=HXW z=XV5*9g_nX^cR^>n?zjSoOr3qaV3O7!0@CEFH&Fg6MP3*y6_G12~o)&YxCubFCW3O zlG%LW*g=`ETBuzkzPsFv2R^;`lZnH_L^>_G@ym z94aBw0P`)y>o0&T6-Bc$Ix~co+KFKgfD(sryXG8U3d8?-&s^h)>2%;FvTmVcI{-CX zu>R#qv8t}WPMZXJ_u9+`-Xdk@@<- z%hGRWJFi9d-qcRwu7+jB70A_!A9dOKMf&Qks3U8N97oQ1QXehku{B?Q`JwSU0*MWY zS8{;dWjTO};+!WrrZeFjzNC|^q?hR9F22~(Vp1)En~#QcM@QA8Yfr=pswIVvVsEY- z-TyH)!Sr*Og#H*?tY{qzCy^)|ASojsc*|O&+=T`{>UuuUN}0*~kKX>86h7~Bh~m!s z%hkN_aQcw0gpGss6y#00 zzt^5ib^o1`rJ936H5$n3aZgIU@&^tDwU%ewzQ?5uXbMO0UO?8?0f@~xz|D)bdpN=$ z-vn54=Rp=3W8Is;Vswy;f~Snhg~?3cxxDV(DTml+doJjCpV_v-fP+Wbibm+RXTYyg z3qadc##}3eIh0wn`gu&LoFVP=U{oB(`2g_W7q9^c;CbieT4e7c=7WSK8a07~<}To= z9&K6!QZln52W(13r2;RHGr7>EQ%vudL_Igbr7~B4o}<}K(GlG6?`hk=CQE?OrC%Om z5Xu||Z)nAx2H(#k{qqh6atEWv4eVtCW(_kkf@{|m#Mr<`FHHBzUThBYd+M7Z1z~!j z<2C_3(xg!po5>Wwp6xt7c77-$8_b^u0e zwtvy6K>kN}>Yc*IR2z7m<6m&6=ym(D?C%t|Wo z@#}CKm6vfj;JXamo$r+HGm+A9CgNY;_f?4&6Z&k+Exp5;jEG3Az*f$#My*4sjnBjX zA<#C-@n1)im@%!Yn|#0}W^kZ-Arm}n&;x9e1xt>RX>Zqk>SsGZ;pz6ac6kg${vUra zlVEsli}A0~Fhp(Rg~8o5#z0|tv$o^!H?ak$u-OBc3^>{+-?}8x@NO7^B$DdeR<2CW z6<6#m%eCM8I+hV^P&f2KQfs>k?|!HV@qm2*{bd6DO-Eg%U^qtliEWNaEuU^|Z1872 zP%J(1X;=x)W~9vKfTr&LsoD#zc*J+7Np(|r*PTNhA%b#;Ck-~AcMY0^lyCf^wr z+JU}s8qhFFoB@W5cP}ib*cZ)yo`U4H{O9i;*Qa*DM=)BN;BHy_tMel;jjVgPJMdyM zpC-WG{A*A-z%_hn35$o*9HKI$E$stFN}N)GQtAc~9mfY32YX@Fg0F)h(Qb#)HKu9u zvr~d-lJr7CGtAB3h$U{22j}8TDC==|Es(F)b$4NOkb_irskufp7X2=X#dnX3Qa{M z7sE(pQnP2!vfDnyg*S}_G;QOh8%@=1Po7`0{xLA>D22g+(fD$UD@PR!-n143*8wbT zzZR?O=rpic9$2#`TpfPf?1-%JQ5Og`zr_}S=xdNuM(?VsTBPlP|HqH=~t2NY&(d;mr z=BqB82I7d6;NPe-N6*SJ*<=#y1$h2!aPzKGKqVM4g<(5a7#2^ zu7uv-rx-3{w>grYD+$+?aP04ayO^oy8OPmqVX6BJxTxP4>dIxoA!50Idhc)FUeCS` z=#Qm!FdR9iJkuDMEb>m5z4?0c+!$F4+lNGkDkoUGo$rtBKZf>Cq`afxfY{bTD)h(j zqBV%}`qfdITpc+eW;l#gt?w4-VIhL?y0nENBn6ypRc0XrR)Y9*;0nGU> zTcQ$%PtA8mBgBAj5P3az{#_yr(Fdac9%& z-bA1%xjeQ6D&uGCSB4a&@1{CRu_Rb;LC%VP)W8p$c=+9yB~Cg^Km?6w`6vU`^S=b& z{l)yYy<+pDzk?^jZ5P*}yC# zHhCIvNS}DC=d*@v@TAuVrf`)t8;X-wzidA z;M(xW+z(r&OI~h$wpnPb+gW@Fiwg%sch}IXt8!8kUv+K~O9hCw{;WIt(UC72YY&6P z%j2cal)@5!K_Y_4MnXj=leA9=;KIrXsM+~F|>l1(GQQ50PS;~5#ZnX1NK0hmLC3Ic|r zJFRTrXBrmvulSB(fw%hV9h`bx5Krg%rb2DgJx8(&xTC0}rB1bTIlQ)Vtm1MA`lt-^ zJk0kNy7SWJxNN}&`zolXrgI~;0r;Yam)FVBRgCXHR8W5{2rVZMC&Z|M@mC(JaiO(~ zNh(2d3{Xiq=^Wg5myV!QF;L@?CGRN@5cJ5KfAd<0b?q+R$yDK~4+%_h!r#|How}51YeO=0S42^QBV*!KvXGGHOd`U$`+ydTA>(-Yu%DTGlZt4ANTwb`EnGN zvvuSuwdN8W;F2N1%Ed?7jpLDlUvX<`%wm-B(1C+_4h$x>IS~SV9)OdKokz`L^ zT(j`WpuB@St_!#Er~;knZ1KVLKcQ)lb=VU{<#!k*oHQTNx%G+ZFH#>l8)lyPYzd=I( zovLs*Xs;KfUih<^%r~D{QP(~}?O!Oz(?+GK`sFZMGRSi=XSlsGbhyLTf(RvgdW{oU zHfQ8(?hOhvH7{88pFY82myI8~p_l#c)eTm$M#7j4Lq0D=Iq9ph>{Z^~^~FwsF^(va z(yL%zTI3NbJ;~9exE6m)33iL9xNAn|jF9jtH7PXC!Ea<|qVi9_d>aycluY<=uVR^E zhy=IdY(gi$giA=bPZRo-cgu$scruzQEm=MVYe6`6LF!XXN#wvk{dsjKzrM$lh;bKn zj`o*}5?8Fx-xu8_6^SM2I!*7^d5!a36dqK@pEO>7YUz`Y4RS;zp@ol67O)EAzu*>x zkMO2lLgD-0zr@~9+k(nJ5L9_ktIn0I55Uf$UV4@wmw>VAa|g=#*Rq09c2ff#u9zeV z%zGK1GvgHK!M1mD-+qIiB}2fMR;Fkp2G@CG$(%jnhDGAxFtm5 zF#d_@K~*~Nrbs(;V0pWPHglT_Q#*sB7xiUWR&?kp2VTzz>|+QrVM&RfitOaI-gQUE zeG&dTk3t7@i3vLMhE88f&c?G}+v&tUN$X9Fkf?RD7DITX+0sNEcRRC& z0nSeA!ZSyRBsO;uIF#mJ51!IZN17+(CKDbGs>?389SD0MIei}UujVa=jmozd~@$?W}f!Nb6sO9PEQ&eo$r4*k(65mAkD0$RrJw4A|iyVS#BVq}RD zD<#!4A5uiqYSZU_n7W9~P|q;z5>5t)ZP1RJZX0W*tB4Tpa$Mo|KTU3|UyBJ{RjU^@ z?tAU%-3@14_Y?kP7(4JhgirIg7|AiJG4$GO>=-rH61w~P_^$7QS@Zyf+YvFX!)lv; z_sRj03{LhZj_tqy`u-G-d&tl&gL#aSCifR8X1__zfwsA114j;KDgILDz2@7)QQTLM z>G>&e&L9F#Nx~C50+M9PxBVvvVFTwg>U;bdGv?om0OIDrEikq(orB@~W})-f;CbE9#p;b*lv^rA5WbUc$W0jr{%d(L!QslpAA}Z8^^QCFinXzu zdqhFH`c*5RpU-^r)rLOPPfTEeuv6Jt>~wq<@UG_eHDO%SQ=y&>o`f>mh70@{ks*D& zJKDAW)Pm>j_c?0fWL5TLb_3s(tRFY7x|NP4IiUuAh~l-Jid+^tRugq9 zElr##m9ICvJ~%L9q7&ih!@0vAnKpO+ z$g_g*aPrGe1543}hF1rPoqO7{S#{J)bk+{f?S@PL>sa-sRi#G(*lL!-bmHxi8-q{I7f}+Ta?fo`*`Yl%`l+p>|BOE(?^6Jys z_8V@isch*d2BmNx?x`j@GQBtU>EyB6jxP;2o+fA*hNKYPFJ#3J;yPC%gA}*D#>Ww9 zN6`!_s!E&(v$ij;X>ke(H^io=LmbmS#ioVrLWI71oSOMwmC$EdJYh~BJz{IIqs-*G z^YH>PNb_^a@t&36f_CbAIeMU(fDr{k>sxMkepAOM5{pBfYhQG`-6cRb9RJ=+j(hh6 zb>%3^h(`^m8mHimC5tLEo69$Eb>5qKvm5C==*o(mFYJ+6 zNeF$EEQZdIellI9(EN^|z=qssBM_XTgT)=fE&n`4g{jLmv?s2Zirn^LJMl32wd*L7plM_#-BXv9~ysF2FP`C-4N7 zZbQ6+Oo$1)%4}Bo-qL=E9tA(xu*a;Db4QK+xlh9Pkm%H?5Hod&ifkN=t-x5`gsxSp zlBg_sMa^oQIfZCYSR#7HV_PP?(u=AE5;-dJDIb+=dqbS*>-srGSIA7-xE%x)yHeSx zSUPm|?KjH&5bMpaqE-0ZB^-}`N}=A!tQgX)@|N2)RO+KJky?yMtkIfbz+kM?vZNt( zpszF?o{lj0n&miv!yKTp1=Mj?JU*6o01Ki_QWRh1Cg9$0iM!aO`xeXR%*7BCl`J2_ z-&|U9k&GfCNHnO^UJ^6hQBGFz6CUk-PG>gp`A?%(cX{B1SW$DmNWPqrMzPx_JlF76Z{hhaVbjp=f<(VY85W1!qt{`REP$Td%Z*89khPS_lzsD)=7m>3>5pE?|t1oT7rpw%5GIUnv2svL$OIpa%d$03P$3V4Ln^ML3Bk!kE;c465!?ah6S81fr-BmGtw1?b2IJ$FCTR*2ZX(#Ho zE3yo(%Hc>9<6jcPaGP%IP73H{z&^SO;YTHlCkLc&{?;V-cAZ`=i(($X;$?-FcS%Y< z9`R; zrXU$<-ChmBR#q9{mlyYOUcP!fxF}4cG$+})g$*D5B5`T}3@*&3WOglCt4>Fk4XGVt zNaFoFd7sUApcn__SDIgJ^=jQ_@AcZBNl1_@;6XXXXbyl{w`{FR(%eaqiZCpm-ewA@*sn~X&*u`-c_ZKk;=@R-%$fFQ`;tUYCdt^Z}^;7Zv z?^ZxxPeRY1nTFOEGw&&-2pe)=X_kA3U_;uV?PEG0ll3*smIK~!30WN* z1)^V~%4VTbfhQauqtvkPKc8^nC|T56ux;Cu8 z)&5ZH?#1DwNR-gTfEu^fb|T}?iCWa-4TM<4jh%(W#Rpi#%DmJNzF%lMYg+%Q$ZeXP#M61Y9Pg` zm2SqFlO38*^*%CLM{cF-2X*+b!?ZWs5**o$q0=44SIsQTHJ871<}|rQ?YbU|-&IF# z2*t|#XcYdk42K-lxWsIgTGin#0x6wWF;;?Yp<+tDB@_ zPOC*@e={$QN*k+DKbLazP#OA^vZo_yhFT@`HW*s@y@1RMK*l&L^%KS=5 zV|nI$9=r9Nz(%cBZ^-TQ1amB-X}ZT-+#svXmU+&!rVnOvzfXj>p)Fk3k=C=X>Gal#bIT`1(0Ohtznn&&@nitPFZyaR)GQ?bd*kwxPZ#^OXAN z?i0=MJFEWmKS^c5s9U*B>f8m?zX-L=sDCrUq7&mGIw`!)8Eaeqk4S=V7hEsMc_cY) zmwu)#VNXPP=})~IU`(n*+bucY8k612iFu1MH#Nac#|PnIS1bRRn%CZnOipH-)!EQk zW+n=Gyy+(&f+|!}uG13L#pmS7fO<8ha6Bf_2eu+mVjT&_g|(EEC7xuY2v1wP-OkJC zCv1?pdi`W0kaOF9Ve;6FFbIn>!=oEi`kmf)wh%K7jF+DeBaWv8tu;{jX64#xEXFaZ2XW$Vja--MLsNi)HvXgIeo4jvk>K%#}Wkg&W+ev zGJV2l<6{a#Uj~uC`2~`%k##Wpnrw!C;SX1OO~xW#Au+XjMlq;f`H33W6#p2RMLhXd z*ZtotfOBQei^XD$GaWNZ8A!cHdGbv(U^r-v`%_DQ9!xDHe6Q421*VMNQPh~6N zdZcxC*}by044g(yn@Xo!yG@f=qJJD@Xg0QAGu-_BCDu8L27@ucS7vJnhwZU59Bf{W z^~OpT6+=l?zyBz%y!^hd!+*f!hRBqS4G=j*WMq-A700FYJqP0J=if z-(3|*jECL1CaDZ8msX?ynKnBk%f(DXm&Ttno!6kTfvs|ySM$VOrDDUx4K z)&H|bix4{zf3$+}Du#ZP@4GlxAsAu`$PE{~`TcrCydOiU({K&w0|v+D#GCENZ6_+! zg5`5=#C$O*;DCK85ClF7V`D1@uZhGg>5+)#ajHZ!U;aJQnV`aIZ{!z-|0$ zA%D|;wZGiRAI6Ow9ndy$PC{~+WG)w*?AkKAa^esnrY7cn|7cMXx*$XaiPQ*>vqVVm zuzEtj=lZ{0`@eelXTjj6z(*X*vHy2F4?8JQ;PawtOH8y8rCN6XroR2R7Wc1ziAmCc zC$8j|gJGw?y~N-2f&Zul;B}Q(2;h3y{#kbYd9DBb z28_PoH;{gG-2Z=Hgc}ExzF6&#DfRTP7&ibs8I3mnH| z_1fFQ{)+8i(r!{MG#KnsEqdLW|M9E(n1lfqjV>Wj+M0eDR!>(jvfIM83$}y*AP035 zHF*+js+#S5u;ALeAW!M>6x5;EU?~vMKKKr0Yq*O3V4w#fEjW0MO}i=7G$QVT&?@Y5eeLiukmS{^VEkTKC=tspS980vimRFIM~ zFzO4^G)bl}YW@@ias-Es1{Mx*WTEj^JaNw7TyMaTs(nP393QqD;1H}>j{B4vsA%|d zVsMbpHq<1|dAOcdl?{b`7R>z?ADggV-GRyBGH4sn(Q&~V8UjGcxnNI4r`Sj`ieLw# zw`mP{i&Ql9eIB1qet%Q=Nxo3D187wwSn#l%L^T5tVHJlG&ey#-Y3~!w5~e*H*WE)2 zSSshU_YAy113zfZFRRTxspAj;)~RdPREm>1EOf%eE}wkwpFPpe{`{M1T$bRqrgv|_ z#gz>{_iV6^m>=N+I73;<=>5i7Y5?38%-0OSQn>zw_Mv-ZGHmdBHJ15D)ksR6z(%=~ z=PwwdCFu;MFx zX$FGMH@Q8?nxN4sSPn1SSLQtBFl_y8`GXtomu8J=-jA+1)Hf++fn3?D(0gz>1fsFl zQo*t%oxVf1!XW{VVbQd9khU=kYF2`k*YMb)9}z$JJ(_tmkTjie7lOxtWRjl`P5$zd zwRXvOneyRMHc4BA%3X@eO1s+fo3>p1qF2zbfT@6_E2YF0J5frUU_6qZyhJ?#X0SNw zI>+wahi`eOs~Kd_aR`=YBq-du(cwjV?FDB-fPX}b!5zGl+S8te-D#^q1jyvqlmYse z$_ZQck*IqJcp~7AEsZIOs<`E|x#4(;CXe<_vEc-$+4Pv0xm)p!=M+p+7R5+ke?^rh!UoS#uy3ID3~x6ynY zq*sBPz`36z=MXqhzNSm`XhCLT+x)Mhzl6Ji?QH?TCBXOb&o6O5cJoLL?IPrRbyJiw z``o8YJUUEvu1VWc;Kr*RKZT)CM?Igu+j{-M-SMSXcvZS2>UYIN*6(rpwNp>jug9)B zu3N53)UVyOrt{MaPaVcqZUFwsalMy5@m5$fnSfQ~g<{B_y}L@Y$J?q#Fb=+Ay>S)) zDX&wC6d{I#NTmatdH=2;dfu=5%XbzmqNqCY6dOF|mqS6s?eshKpuj3F=fJ1Bg>EQosyaoX4G9W-YI>*vW5Dc@8>qek>^?9 z@>NBGHya#lT@Q*S^JI=+nU`A4aGvZ@oCyWb6cv9|U1HT=Pe+Px@0eQUz7G$m_! z3KP6v6Q$3y!V0ln*k=;7z*B`9Vi z)L=pS#2Zvu3fElapIf~hl;`$fF{HTFWKtHxE(qX8V|0#M#35=}iDz1a0m_s7_@K9= zVAxASfM=zj(T_L{tZhUxkrc%c>fjwPJHi4Jd_}-vKpv;q6-Oq|T{{Ll0Nb=jU$|dy z6*!J!2>!$-R@ZfWb3Q#6BrLh2Z*K?Cwb>pw=hHiRyW@`cPx1_a*1Zhi&QHLO_!uO~ zEx67=?q#hO&%4u9DGC+bn;htumEaGD0fgI#@4&B$pxKYn#W@_5ItpZRI$7UOGW|L1QBxcs;l->twlvpvfL-V)hJ zVzx4}6jXF5zCISoF&dOObo^YT-?>@nC1G9HNs`>w??QE0=s;p~DE)VYxe)&=W#USd{tR zZkhmUXDi*@niT&Cx&YQ6-@`Q&#UZZ)Q~2x@3;%2~Q>@{h_-&TNo({FnpUee^VKZ}^ zQZ%q`P}P{ul6Y+25=c@HZzWvcbBCM zLl(dRO$I#NhJ`QECR%}sO0IY0RUfaiP4itX`K$!=?pMSJZxri$kjsv>>B;cnY@35q zFayKxx(Hd9P1X9O+v&B#d$)t23}LrG*uKk~KOiRIUmXsxiSY*`q|3h~F;T7ZREGM* z2Bx!Ie4Tz&rT3l!{19&%2)1AA^pUPtb5rZeGt1q88jaW!yE(ua^%g`=epSx$+`C(G zIUa~U)G_FI%~vgF`C*}gqu-Vf9Kk&AigJ4|l=DQslBo18LDwTi9|ll}<8Xc-hY@)W z^>ho;cEswQTVcRU#2M!=FAUyzL$xe1$f@e=kAr04mK7+H_LVnn$m- z^{GXYY5RNnzNh7)YJT4Y51*l{OEF9J4@7n)@xm+_YiDWL?$ZwDf?<^YuLt!EDRqp? zzHR!O_3VO5c6x`C`0LT=12>K-A2IclJ=K~HYCq?B!XJ7wrt$G)#7Fj-b-?AyCGmNS zhgi-uQ#$!cNOZU?(hM=NkZELqZ7<5VZypo7y)=Sn(V{U_xoImj{?5G85?VViG=i;! z84$k*PTREHRT#twN>rK=jT8;l(K*SVxpm0*$?Te%bfCWC4U1YQrk**S?ULWdIg^*` zu}lJ+Nw@iJm18B=Ul%c(5SUVXa!TNuXbo`$P2e2nzN3vrAim}vCmkJp;aaU}`uhmm z_3NI;oHnCM9R68`K>w5?q=jo`K=J4X6wQNs&#IV=x~4hhzZmg_2HmGjwN99-zEfaY zebx3$Zxm}SoeKw7Sio0=ok(oK#S}YnGL(uV zjg3JQ%bOfPsKrQ{W+UXP&ur9o8E}Mo=O_ii<{kjIY;N~jwA=d?juYUA{8Qt1fWP-- z62v_SHq#s;s>{tUrs@nGBTr{8QaC!n30``OawAc%e#YnP6vAQD;I#MD^f=`yx8>_c zHKTO9s6Z_<`7p?l9}r{ zzC0NlE;+}u#sR+9bjp#r=P1D@fw-rJQrbJK9NlC&QjBN6<+SBcG9tOKFe> zNr6QpER=4g8$|EBetVyN&bjyg=NJx$^7W19eV#c#(<=dmLV{j-gTKyspN{GppB9@l zw*lyxo{MdFzU8l5c=wG)pl24}7NjV%gkpu90kyf6Ww_{Ur4I8UQS)oZ<`~;`8%tdy zv85t`(qqPBLzd&J#Qc`#w~H%wJ`unQ5`Rm6sw$SiC}WksYqk7Ap3=fm*-$Lm3);Ud z7^j!7b1tqLQaU=L^nSA?$~Qr1%%)R)XX#8OCyjaL6iC|k*Sr_&)?xXk1&aAWIKm;^ zcVtDmWtQHvwNZI|iUnqPwp9!tZLe)tDZ^6rZ8!Imm+1t$t?!GhD$U5+vKQ&v@uE~I zQSB5~$3m66GYY~6C?tP-b<`NSCX`)ITR%x?RwmHvtgulw?w6M3RU+mSrmCvLg^&`a zVQt~VVBUt!Uq@6p8QJNOZgzqme_$$A^4sB3cHmyKqEf!Iz-RWzlx5&F>w62R8((qu z_DP*VYkSF7-omIyWQnjLqzHL2lIc^P;0zO!EBPiq6K&L3tQ~wiv4%C;VzNdRlULk^ zEUGVX49ORzreo`7-+$PS8krQmXMdSBDyifJo=l$z6?3W`$(K7=46m0eAZY_lI#+Ee zyqt_!$x}s#*|F)=^xR{NuAhNJL=DRLQ*zy2hbat(w4p}$h1Y3Ur7z6plv8?&G#R#4 znsP@hMH;N51{o^XzE)*)t&Q0{4i_GJu`>2Vp`O+0UX^3H>8wR~lS9QO=1`q(wC}r; z5wzISJXo7TzA{SP0Vc!~J3y?ynIrD{sRnCQiwlq5HFj?MJK#c;?VA=c<(HT}$A~gg z`5||TLZwuWZh-5AJ+1|F;ocRAi3^sP`T9V&YrvAC!+Z@Zd*RVn_ps5z@OykQCif$tUQBJjFLKTCt2q`%oWMuO+<$T0>)V*;cSUaCbr&Ow`4fx< zINz4~nR*ko*ju5-09(cKvj{f7EhX--04=Y=1=|rZjOYoVQ<2=%}QU zZGLSl&iKcm-91<}+nnHat6xj|YeEel#c7caqh+?Jr`_@I4*kWP_pQeoG`MZNf8mHZ zt4E6w|FJszJA0k5kW_bH*-d-8KB-@mp(wGXMpcrs`j1_a%K&z`#& z<=?6@A^#=H;I#)zX=tf#!`8R6WOsV5ZB+ikQu_|-Li}FF+2<`+K*3pmMHF;gLC0_n(y&H`S{vdyICDQdU_x z-xv)ot|;!&n_}Ru|A=&am99Nh8rt_A{m*lA9vjWU12fd&aOOvlVz{g^aiQDv>Cy)wJ+KmC94nBEiHN&47@cMs zae~K0$9^d-s}1nCj3<~(KT*>XiN_y@J$0#MLHd2|L2}N(#A{Hju_CYYauqvMYfe>I z0Lt9Z?t=0W=M{v-b!B{q+gjmFnM(3?M$>ueT-Q>ljhuDz)gZD4F8(uOvH7R10lO7A z$?-CF4Et^JX=N4atPaX%Y;`;uAB8AzT-L0()FO`qr6t&C8%l4h&cB{Gm78-wzK2~qmoAw8J z)q+VzWJfea^J{V+#ntg%jGIZ*&nCV%En`uW@WxXX7b2dHRP@BRlEq#cqaC<9^3w;b-}o@@ZW*R%E=fPE$EPpire3RKco~n93M#%`G(&CX ziR$TTXG?ffw9)x>vujN483=LLcAP!y3KG^x*5*;}5Rz=2)euAB zi7)VXwVbboc@K5-9(LOH9e5jjPU{M5(dK{q1#sa=;QhN7eS$rM!WGv*r!rkr;}{C9AekF zCBgpY49AwhE6(fGWxUiZ!j0`=zIP;(bNnizE_o$(4II1nGl~?oAE)t&1^gI`Tvm9r zI=PEu#~-psYN&G!C`(eZIB|E;_(_mm7J@^NV*VD}h4uwd;enEvrpWKU#!Kr?S<*P%9wK3QZxm}dDyy$b zw5`${(i*8}RC-322R`;yp(;VRmd4EvniEon%KGn)V@DT5H$`kBNsq6x|a}Jh^{phHSa~&I}nt^1i zD+KIUgt{rRH_Q^Hn_F!_n@2I;#KBTsl5$_mqDzw6FOmsu7w1eX*UppRvtsx1f9W0e z7Lzw4YGrX+o5jjlB?I`(Ae{^t_I8&ADev_$a8faJDkPrnl9#S&Yxp*!8)-Z|VP~Ub z<=7BY%w7B;ZW$Rj3pqEia?MpsD%_7RdSQZ5t%4)=+l4XV6S3keb5V%Ha&vW@#T2v@ z*YS{OUl@=JQ%OxPW!ROg1e|(Y2<%p?Z)m8J1wvoQ{dg1UyUfQnres+2tc5>M;yo3f zf^s40`@}ut^<11-6M2WIigZymt`hVR_fG7ML3l1U^nG-59o81v+!AiAYW-%S#E$qR zccHc$r~e9j<5mP#vlB{Wd7hU&RDy)oXhtb#{5@WPp`7!&L0L%o@_Cr0#X>@#eO5-g z$t$IPzRe9@dPI-jn}o(WgiqD%nFJkrDkx+kNaQkdsOzQ-s%e2iz6rG7Vl69~+`^ik zRx%7I3qIhf;6VlKY{mrWJ@#KnYZc=E;WDsRlrXX_4D%Sp&vFpll%mh_AHug4N7ITo z*zu}UgK=)>E0faBZO#iwWXV4g;PxKj-RGlI&CXGe=2sSIx#59Na^7{whG z#78YGEjMNA+*q2uFt8ddo#%E~r?ct2?OL&%-;C9;W2uT}HQ6+6r_gSR<$5ZaxW*JA zxxoHN`Tv#c>XXYjskPQ3eXGrut`k2B&e1Aj1hvnQjdMEYsHR!-NzG2ZfXdDIoM<;X zsQVnHSIwXz+3X`?MUhdXhT*ZuogKg_ucQdXI)Eq*+Qoc)WX3)qgub1I`t0+wUlrEJ z77nh3Sp9Zf`2A6bgWSqTidt$nfb}q6+?JOh1?&4n=v{0w;1P)aU>YizvRzuEB*z!; zmB`qeFj0~m#QUUW9m!I89nI>5)P4h*(Ia9=&4R+{vhDTCBX+%-2eNgBKV! zb%gilu5SDiJr8Nsg&GvtHi+MLy3M_xyiyaao0A~v-oN?+^Zf3jdqEqZdpOU8ah487rC4N?^ zVL+(j^Y3ap$w+sRcYmlO)Q+f|9B;H~bHjn1S=da=>yfj+izV83O19@UlOV7AzUpG| zdPwc>33W*JmjQHG_r|%(@k3Wx_Yl)l$Fu&kLIlD9JN*Lvh%ye(BjC+d{?7AIm(p${ zzT9NjW3^d_J}GPW73Jd&wcWsrgZ}!=!B_;>K05lbm-DJQi_xP>yMvtNPTLlleL-Z` zW-(;t=Gr3jZQ{Ew{Djdj+G9NU=K>pv-l15`k?jxDR}-b73=%$7nECvgwK|V}l}vMo ziblk>%3PLWySRo%?2zgev$7t$whZ4{f7PzKI9rn7Mzq8@0nO$WrE_N@#mcZ5PTtt* zZO06-SYUBnuHF0FzLz-D4$Gp@C2S^(1un&Ax zb|Z76IC1YP_^!S@59GH!uvL3yFHoRKZt%knLtk67@_0tyx}0lJ2LB>TixM2kk4qab#eoMQXg=gbh{H{TZBpkH4o`FYwy2usMP z=O(6@`;6S0MRjX;+)kSW9FgDczU$eGn7M}YAe(UqJHBOv2_mD)o1+Q{PztvXj_TmL z649490mApkX-Rx&x2?N*pA6Q6=iQmFjrp#wtMHXwG@6{g|GS9c^+T_TQgZahaDpfyC z-djYCKVrH-4GSG9Q&;*#h3f39sNDFpV3C8*;U|4S$^=V!UXfkj=N#K`JJHY)=vpYI zv5W0cYY(_cM6HXlqS1xnRIAFqrxuV&Pj*%n*#kcbk+?x`?i=ofy&oN|adOJhc-Z8S zWSdB#*CP0J`Ud0d&@EI{>k?8MofxE}uIpER!^(Ubaqo`FCpY?NK3%?D^?O^^a!ou& z=qGT%zrSv>=@ZMRGj!TOWHPHj1=>M(4s%EwaRGBSE2PQVU)xf+LHjyYGU6)idRZ7H z=L_5fb=CGOt)UxqUz=8fC2-cl}yU*{c4FSehF;~JG}a=&-R{b zo$ymEphE~7TWu3{Aa(`bedh7i1CtSf_jX$9CpaS_)gC$J$sfw!)Dv3ud9YD@AE^Hz zx8ud8sIz>JS5*9e48j3>lUMI@)hQ;wF@IzwgFFzJL;s+xW@RS&TA8`(BHBb( zcef0ly-*ST3f3j!3C_hnUCcXY`4Fwt?u-F-m3m$#-+Jz=xaqf+s|OZm2H1YaDn42o z6w&M28ph4oMm7G^xS>K-Y|1UT;-4hL1fR0p9?-5n62?nza+UNuDA97JldD|zWa*T; zieUccH1Wd?Rgk zgm7MFNbN^fMrfre4>a^Fw)BrQ+L|3Cd4F`IUm1LE*K^on)F|`ZlaxT8dU@cq$Sr@z zlgr0qLgRIPeq=0brt;XIEykH@;c*%Lb}3z|U55FkSt@$*>)#&P8nROdUB5@!i;kpj z){xq_HLc}53qCwM-0&{V?B%(y50d%y+Xo)I zBet+p+{ZMI#kUV?x7+h3YUvHi<=R9EdFPqNMdA} zUOE4=UHp||K?_0G=GJ;(iq-_NZtzMM7;=<~-R zw)Akv5AX`Z2dB30(IU0OMc+@fzDTB;eHE!-QqlB9FNc@p43)Q>eL`rt{5E0OIObAM zRh@Q~@7I$g=Cd@Wj5!ssmdIx#&DCL!BD)l<0Sv+Q$+6^{t}iqY2~jh?b!^5Zn(J1+ z*-$xxSTSY!er7HPy_XK+stgBtuoK19qE%~(K|6XJJKRH;b%?fYmg6d@2OW#G4 zoTI4_)lV>6xAJ*Y`v`JGhkn4IRu!8>szssfJ4^CO>0-!&^OV-l^kithGNUG2#Fkv;`dM zmH<2o_GWb!Y`j>!;aky$2?RC@#gh>At;XxbSnjsvo)Zs`GnIiUgqo(3KEM*J1A*no~8QU+q`^SydzFy z%W9Q1j@n|YNhmhCc$Kn!`t@<7brCFyb3le+^VH&>fqoTw(1qQis2YT(h?Z7J1G_i+ zhn^<^cVoQI`Gk-F2HJcH_hvY^G#?LckhKR*IkRv(z7Kg-eUuk;&*xYN71>L(yVmYI zP-@T~OcQ-7JDs$)Tn#w(i`a>guUjL&oAANn>^J_r%_E2ug4{Asv3tMlNi%^9DO&S zP7=H(=G;yXTwO(1Rs02Xw~xA3dFI3>rV$Sj19DeGxSq%Gl)>7KIHP3^9HO z{LV$jFqup;OI+Zr^}QlsT{GVjF$OB5s-Wnh2^F?l2ISsCv#Rk$vn12GijEg=gne}K zsGEZE)hJ7%7U$<@%f7}bxMU@05WYC3^}FDi#``1w3!uJbu(l$AhK>4_&%sE6l-fEK^iKT(J18LC&ZR}QSA=sHn9~CypWJVWkmS@?JEIG42Cs4N_ zlXnWoD`X4DYPnJ2cAE?FN)Sl>M4N*kn!i@*vWeatp~Ip5(&nm(eC0g}S*?b&r=04}@jC4?!P{nVbUBfGzE-E8d-V-^+c&mn}m4B0$v_L1y?`6>7^TSpJ=Ku@%fRLh#4aYEpn%&+8p86gI~MnGmgX?E*gS%sDQdS)O&# zc@AdTt+vCK+dgLRafMc@i1h3)R@n zL;s?#C$BQ16?Y(1K^TxgZz8281FQW_VS{N%UXy1q!l80@-DpzF8T6;TeC$FgVJk6f zC=-#o+STlWonPYnVv3IE**2D(t1o|Te?Z-+OU1fd^}j!43JtgUM_>29be>hzK_r)n z(tIM^o(JDV;w|>S9?~m3_5SLxw>6ra8pucgm5N@O4}F7$OO3D*c~Q)AYLHivzdjt7 zmIg=^U7GEN{eZvdqo`#tnmPu~Cmj4Iq~(ml&az}}u780mlj}+zgFiBF!%Q^WzLHMxrm@K9ohOy8El>msg7zN69YaBm@ABkTuq3FH= zbaFGqM1Vn5rhy^5qRlnkU;6Ptcg4He(`XiF;Pj@OJ|R4C0aD4ut5cf%auM}!ev^BX z=aofL;N605_ZXaY16Q@qyE19$>i-eSYJEeSt7JL?o4f`pJ*EBZ?)tbI$t5O+ zKA{5));}GczXd$l2!Q8(%HlHPw1-hEcEh+ipe1z&@ZLArB9!{uhwmnuTBsj_?@udOFk=tw&iO;!6D? zySvy=_7kLW;^_+{IZlAc&2eu2G`}<5n=(_ua+k6^FD~jrT2c`uHFlAK_zOJ;CYXFt zPx}S-e#4EJTDgcTdrR8ldtm-jlwKwy#9Q2tZl>kg3>2i;A)F%Uhq?g#&-V9M^2_X7 z0Le5bBXIfn3C&^Ek`>-ZiW0XNlswf#d~Qd9S&fjVaus89LTFeyqo0KlJ%f4yAmYi@ z*Vg&cL$wjJFYF7Q+a~2s4g4$o!=7gd+R#D#r-TTO$0f0w_hwAO6|dFMWUm@15GIW8 zC~U)5SytD;_(Y94b<8&f_3i(wjxJzFw#Gjs5E%O4*D597ahal{@%;SJI}1VCni?`n zVQ$!Gyjoj~qEtgA7p;c9qUeKOcASVt?vWX{us-U^p#emg=q)m3XW2w#I5|bzPGGE0 zox2aPl!W;=7*o5E8h!fhPjp5WYvPc&5hOtp={kaw)=oP1*7vo=Dr!2kD9W!Now>rD z9%U&(w91by#ASpk{VT-u5SanRlgz9rh!@_qw0W1#dTkkBDe~k5=#Ha8Mx7eIuB`>4 zQ_(xe+VV{pwd8;Y4r}Zqlajxq$6##=)Ec+=9#^^xvsodgIt#ElW+jsst5_VKzNAH* zb7!*K4}jx#VUtD9ALO`G2(yY!&tiz(YjW7OurLFNFOGTU7xY6uR0N9&a){ zA2%zt)V{e2V716;5s=hM`u4C0*W<#^3tLvSlZlX7%k6!nnbpAb(Dc~S(b)IM9cV}pcBWTzf8s)JTzreA0OqW@{bpb}6}ce_eT1YgDOMW@>Q{ASI;(h+;N*E%aBgVyshPWdNDYd8M)uVF zRh*=WAEuuYnyb<0OkYXIBUD=d3Tua?Z@>Thqd1}Qix9NdldiFu2XBOtzS3$WwA~*H zzS;!8u0R^R@pb?4_gRw*&!->~&Ja6Bm(t=05~t=QQJ^P|wpSC{;YG5c=k3izM_0O_ zbn298o~3YK5kH&{>YX(2F1we!uMnJqZe$QlD9$#-kMieNRK}gz<5U*>Sz;hb9u%G` z=8&(*f4INa<5QN98xyYB6QoP~{OyJAj*BXdtOYhQ}S)IAi11RZ(;fOFV4$F^vN zw5tZX9>hr?98*P4+%*3Cf35^uC~yK8S(UDW0?nlX3z4gRuc$84_S;P562LwoYK-61 zf`k?`)zc3@zK1dTUrU`7*h>`#VP#e@e8j4$PF(DZ<59&o2(Dl~O@aafSM@Xry`k-g zx3Pk7l{}>rjhnlYhB!YA>wP~Z76OyyU5>Z4EztNbDn=Tk@%r?;Mznl@u zFO)J4&i`k$ev!hnU4Tk}h=I@fUuUqi`!gWaWo5qwWpfw+rbieJTpfZO3=*JX+vPUx z4Lqq6RtQ1!K(9LWKJF?;GJKC#f-_()K+q=i-6cfSx+p}{d|>1Rw4Ni*p;E7#TU1!a zCMECHnsHUojb5A4ah$Au%)mUVu4z!|6(0rJJ3nD*#P4LJSn$t%1cv3RH~H+fo}ai zGyyySulE)laTz!2lUo4ngIJAqiv#wJotj>pVrjm_?@=a75W>|6kl)edy+2@KcEwDq zdd(jCsN@hZaf&tIy1ecl0hc#vAJch9*oXq;j5l+sC4JxNJAK}MVL@p0A$}w@2mP?0 z#K@uWo34rHJh$5xHNvb2HVZrT(M+l8-bz=8hA1|rpjvKTp8JuAeg5K%!o3#=E{FhN z!hQ{bYL3h26Fyo}Z(x1k6EW8SA*9({#q+ybd)>fy*3>~vp^uP{@D-`>0nRj<+hK9a zWr*CSqV`HS+G05FrOi~+57!Pj8m+of@9tT>F-Y)0F#0SU{0tr8+|I6lOUk;+HUzXH zLZkx{{ONWzME6U$m8k+DaFyjy$XNtB4$0rkPXS>7t;6U5be-!!GTWfq&fz72GhT(5 z=e^>+8wR&{_^`aD5~i1UxS0I{Y|^D{jois~0|csjRTPbHoD-)ahIu$$&%cPcv#>er ziSZf0BNq)`zy%in`ajGfhWn?hH=HOk2X$^DHr5_ZbhbDyvqlM5V--|CmpgvHA4w0i zoZbum<}9U;qU02t^WOyxW&QxzIVyhwvBF_0cQuyQZ$@NFShGVWj?GsHwGwGPjZKT- zJ|#s!)1=Cl_TbIXNv7r%`O=coXYzTnI{rjB`#Vrl$i*Z1>}|fkKBBHWv{)2ZTksyzWTsY6fKG5j^#@z`R8Wy zk(v76RAn_LL=i!6(C+mdM6?uewMhC2lwTZ`$BQlMl(8tj@Q3$5UZ|iQxRq<7evZ;R zkB^so=sPkpVm}=PK`RXT3CXgF?J7uJO|oKi29a9p#lUg=kL(YZBL>7AvB0f66M1j; zsOepo;Nh(6#qmriUx$^yhjV5QUHhVO@rS?D1Dm4$*Os&elgp;IzUiOG7j>RL7f$Vd zDE=g?uchJdr7a_%o4SUFB1ak|&EChX*jEc{P{d(dtp9L!QZ;F&wrmI*mA^GBr+#ve zYXDStX2xQ^tN6B}Xm#2c!5-k1p6>rjXnyg*nBN+IWMy90h=aMY#wC#GkCHAoo$V$_ z(<%}%+dyj4hdj^Jm!nH{mGlBEMeC4Q`~Xa~qvE+<23zN8k{z*NS`>fGDCgKu-FGT_ zsze=mv7jL6k?k#-pwyfpj<%>QO~atpJuX1zXF~@uMoV=776d57u!y%S-iovcqiR`@ zyxVU2@2y~L$MTHC@SPcUcuOPvp!=tGo}$8cA!rpzdrv>fr}`1ds8J6qSc5R5*bIH< znb@)KK}`WS%MkgKIAZbR(h2G3DJZrj6dW$NKEy5IaGI%OZUAQohsT?)*Iy*543idr zgQa(jm^JeoTrqBVfxm5TSwln)2rbEWMBfc(2WK$VJ8p_Ei7XF&J8=IC2%OLR3qaPB zH|^Nj2FUplIrdB&1ZL!A?@vPWm?~=+kiHKI&qxg9mGP8ZoiVfRqkOuv#1O4jFsSrh zgMNd&?)L3TG!-!f@(+lzVwpGnK|HH1R!=5gVcHlq`RM7Jg}G$k3`=uY_OfEDJuPVd zU@EneL)|C~TwQjexAESn{&=PTxIvKZ>VHxc{^mKAH{5@-=^&D(y8@#<-uS5WyB~360ux&&PFUK2W*`hClfJ%d7#`ZGp%WZa<7Bk)7}7-=E?axR1s| zxv15YaOvHuIpjPR64N-0Kno4V-FXk~$Xc)~$_<;47hNq=&ruW0BFy&}eemyS0clVu z;)U6|;knLxwW&d=m_{rm;TmaDJ&BOs`|*yLw79O>d0`Qq{5oD+cyv?>mZ>=^<%f({ zk7ykT;>h`!0FvUQu{FcVA3>ejWs3r)A9kcUHk`qO8}``}{dh)|E7!|l(sS8elaUqT zmYL8W&QrqGLTW$(4-*TgwLHVPPsEymh@vTq1zD58-i$iO=kUNR@yuFJw^7I+Pat|2@>P9l7z_T7t`vs28!afsU;O&3QHe5Q14s=wQ= zds8}Owm#beR>kK$PJZ$Xuh+hDqr6S=|Lc&5XV0>z6xtA61+?ZQ z5j>=?-W8&R6?6fMVE=58!)B>E-YWIPA64hF`$ed{psbS4Po`TL2yiw?wPG`%J4Ox+w2 z*WrdynR=|p8UEQ%>9nG*$rN89iNifySRes2hz?$ynOuhxVzaDKz zus29oj@N*D^y@H#Q~IP24|B3VMXvtmsf4iZ5!=A2|NT+%HLTffqs|A6u$SptzF8wslSq26fPSZL;R4%fI3meZ` z<+1P0IRyLkC+RNUkU+tr(fn z-0xZ$JK56*ndoswa76d$x?cVA$0m9xgoHO$x>69DsQf6EF#0r9sbL?u2Y0%B`P49e z1N@6$^SS`%*d-+aUtPIoLUAmkbgC(2g+#EF;Hr~f2zkohk%c0Ea~Q0L^p`J^OQ`>t zsFBc7Ws}{KW$WC4Fc^2WNpU7ZgQi_2qv@|!h-`wRrmllV{PqyHcp^SQwGWV{OWRw% z#9`!2^!tHPWLRZ+ZWR-${^{eT4&XkuU6lP6r8sN68B+_4+h#uXiQORJ-0pAO&%rzy zGA6wJ#vw$>fml5NqiC1!Ql=^%S z2`iVLLL17Vx`n{3xdMIIYO-*N(wxk3X zpkT-bjxsfAPe)1sQ;Vk+@@+Zxmm_vq^S2My?EPuPglgTJs6nHLaNq7PaD~06&fdsi zE$Axyo9=KeptTVnC6Z_6$nUo*7dkb#udjWBBF%u?Zkq5%Y4vF)Hy?`!bpU3~PVW|J zav)HkyZjXrQE*iG+4uJd_;yOxu?wZZ}TK7SXgrWKM_Va)f_YQpv zX#54Tpnt4|gmP9Z!A6le2}5ek$(|Svo61W)@*VlxF901ML52D)H@_hT6f3r|H2D7g zcbO9wh$Z^MGSf>Thr`M;Q$sY^a}jvT^M z^~7;hUTN1PRlAfvr!z$aFsm;p%vA8to>JhEy9z#6IeCvQCkFAF;e|hw6w)xQ{Q>hl z=AGcjZ~Ql3D5Ma1PsSsa<6^L;c#BF?@Uf|Rm|2|)vm%XM2Z`^X(P^hxBJY8f^a2K| z?W``Q77M;F6d+DPqaN=7di3 zHjE4SIEa}m$bP}06cm^9AA}a0*z+z+m2#9|4O048gWO%P&oYx7yUEw@Ia^fuI)64` zJf%0FF3L^TgeL)wvt zt#4XR;5vm$;%1zm^b1$#y5g-e{)rt zbfYuD=Vy&kv=o)&bAL6{>_#b#JiRRpcr|>FIyd8};s+O#I{j`=0+~!IRnuL~r@`3Y zWx2Lh*F^HP`w#qmDl!Nm7~8VOcT{01s8TIwu4u=X#w1m={`-4ZYJN)zEdxV*KftY(mVYER>bOOo z++g42(9@R)%FOula9^25nx(6*E=f`=#(y~l`@+}N8m3+wDj_vNjBqE!$F>uR84Fv) zl4Ut9Dy6{8`W$9i7)HWs`@VYdQr303S!Ff&oTb;ub~co2Hey8YBlz9r3*3MnfoeHQA#W~#QHEFd_%m?sg@euoiuibA z=r7cI@*y{YWZ#yaHvd_5p|B3CC7!YGJ49li=b7HUj!+6tnLbP8HUGJRmJ-riBm3k) z6UAxq(4rt*r`Nv4$3){#Ow8abQ*F9ei8mBuPIhAQ)jH_~<|+tA5ef@4)`5hANZ>+0 zcSwN`3?Iy|ZJZ;$a3p?lGa3T+!MjYl4*GUpziVKJdv3JVQbVeP6S*X6N$UnK>nMB7 zB*ww|96-=OD34QXPgOM^IDN7C2Q+J|2l|1Ztcc0~LhL0U&-2**OMa1m+|e0$c(uj? zJl17w6gn`8+W(xYjPM&T$@abRW$+`I`foCZO>Qg#i>w6d`KLht3RNsfg+%Z2LPTY^UfX z;3%gr0AfmPNgfMRE6EL9`yrDvW_nM_;SWn#9grCZA=xz6q>h^6p4Y&X8DL(U+(`d{4E-X1wBbtOD3kW*=23|&sf&)9!~o<~&~4NN zyC)fffy$5CojZ(u8JGcQ9uZV-VR42hjcpUbo5jE$rCz)CiJ;p2=^!BSpQ!aM8#=fQoCBV&H%6fHqFkG0*^CX3=cUlH}niDnF2bD6FM`=xs`c<61fe00=s zJhtIQ-xXu~*p!rI^z?E)KsgJtJLR7HcP$DmeC^xwU|{7GuqY)@?1;$UFUPXqn0mhL~D-2!A}Etpv| zJ%%{0_=$8cLvE&J!*X>xbpz4e0=^R6qvzEWJ_1`zUQ`B9E~py;r1~e`6V}*X#?-A* zG~+mfkKc8h)~Ygltz06~7$^c3nOoeRizQgXe^^3<12JT>W1&lQuJ0SY^6{-pUd!D6 zzi)Ug9DeupnLR%JZbFHe{(W-%VJ-fx`>*XZ`VJi!JrGF~`eF>^+wsU!xs8w0ml4CExQ3>SU#b-p!`17A zS(6x5L{#1l29BN1YB~ddeiNzhJKse89moaU!#LDh}Gt{qU9Jooqgz;K*hm!_gwDLGYxW15KAiBJ;*~ z^sZH-yv$1bz-uC=x@ImLM9c8;|NEg7dKD>B73Mapg^(SVSShp;#22jw6@G>cJiM zoB{e?Rn`4Ho@!CO6v#UiiU6s8f1H^mWnZpDxjTsi#LDLch85~n;gU_*rq(J}6tT)j83n=JFKH}{4R%D0t}*^}ML^WMo6fC{x>0~@JV zS83^I@kpz~zjuHf0jvuIuvX_!ODF*&m|uGb@hgZBd3dZZG5fJ(--1Pj6E7tx*hH+u z3JS5$8f*NDKA%(jVVk4zy&yD!*k#BIq??BUXopY%diUKy^&5#TPM8FhsQFb$%M|+e z-}-mN|8*Vxhli6P0zkdFBENqhp(q4yFO)qX?;|Dp98{Kv%i&6}!^tJ^Bo^m2BEi^U zK*E?@2jGfrYtcuVnvS_|hcmgl~;@tc}y zvkT%|>$NK%>YysQHLxbCBS-T9XF?p91-STfdqCfs-4`UW@lk7bqe%i!r*HaKQO#G% zZ<n&Nzqs?yAnEH+ke{Z@a5%qrzE=II8>8pp z?#xe4agD}2IAK*07yVoH}9JSM!*wr`~+MAVFn7u zkH+TWWlXWX5w!kf3UO4GW#%{jRDKuJX4SPVGvY>m#eYsGrbpS9svNyUb~nI%ppF^@ zX0*o5hX$T6;-q2YQh$Fh8G+vUaoyQhwupeg_p3|p%}eVYEnrp%N2EK1FGD!X(D!eu ze*=OFB_l2P1JNggr1@-xUs;2}SMv?O|1|)(%vEI{PF!68fM;pn&~bFP;(77hzGEdm zH_KIZLQfBxGK%-tW4)yaym0XlwJ@v*sz0EM)PJ(x`bMgE15lHOzW|v6jm!Hirtdu< zbvG3FU>|=iCX%s3F!up@L43$IvRvv5`WCUl57`b z5o&fl2m>&F1DaYpB^&=mFGM07$L}fUl0+0_iwH^)`H{E5PFZ2DyesT5Ns-=Cw<0WuZ zW^!RJodZCwXKpROp)=rF=0t^@>63kHJV<`)3%cgfT%||gFFQS`UIW<T>uZ~H{^-P=m zm$j!WT|N;XO**P_e?fD_-ek0Eln<}A;YoL3qgo|A0w;PL&KdcyPYg)f2c(y=&X}v2 z{AX$B@JnLq25&Lo@54D$7BNldeONzS;!_CJLkRs*#42i!c5P@-ve-eZIcQbt@2w_d3r1pFsMz+L zPCsM1aZPgJ#W4oDH^XbY{sW;IpD)-#B__JEDE&-n2)o#5e($fNH2432)DJSyZ07Nd ziE>wqOE9AWxL3ho^90tei${eEj*}9q+M+=4qNgu6zS9ZzyGsAC zD{s9}`i0O=&KpjNhxC+lT)>Q?#QTi*2XI5QyVpc^_i^I@g<>~{n#y}CQKiXD5PjW`O!x$G|Md~|Rmz>k zE{C|$kSeVzjrsZmt=p`+8CRz8I{>lTRIR*M`=h1zQ{@#PM@;nu{LR?|a~x;fxkE{ zMWm6ayvNsl{b`8IobTf-ThI@D3lZ#_J9U~Z)O9qj48h9V+ak|YrDnu}5mGv~6?&~@ zMD>6n*x6?4=m!b~x^J?w5(A`;(YE{b>FM6}fheU5e@md$iFJ zBz@z|dcCnnYM^-J4K6=VsY6_IQFXd3$h>*|shTgRV?N_OP}4+f*v1p}OT88;`(YI2 z-pS@m`-Xmw@@p)wO{Wa3!(*R-&hS=4?w|~v4H>)v)em=N_6l7kt-hw@6eI+saKUzb zm(|bK`7B2gT)-5y+WNp)fI;H=7jP!5gZATs>lV1tlBsPBeYv`RH!L8C#kJaAoUt+0 z{Qe9k*4iP3)96c|52~umUd3w`)yJVXY_akm+$8hf@+$FX;13Oy=Xx1>rLx|t(Bs=- z8blfBxdF0E*i})`2d3-Cq{a#7WZ76jfg1LFI$SXW+teiwr4sU)Uqatb@HZdkIR2dZ zHAc?@yHjhJ;w&;Y-*0KUS-`Kr22;ib6oH;{R($T@2o5RHw%-{K3Y76U2xEhEE_5EY zCqLa58d+snlb(up`^uAt=735GwqNRwQi)KOp=r9t+u?!tJyaH-`@t<2V#)xk!F!X? zc9u76`SEDX8zq!I-*RnaMk!5LX737b`7;*5)RpN;s&j&t4f#prPih1dA9nK4xd8A; zZTbbdfC~?}^Ak|YB&=ZCb37=eEzP4y%+5c%_I?M&k|C^=KQBJQ0Xi}aF3jm5z*6uT z5JSHWUdnz|Bx69y!L_o?A`9E5AHtA9z0K4FStA^WQY09JtC)p~`RIkT0`Uoz?ALLR zu>(T8nGr>_KS&c4KiCMm9^=KnMw3IsVOpFRbzpIwzWxE`x~k||*yjMJ6bc>^_K53g zb2wq0rN;P5#@!w8C-!Kv3EsPm+=^Q1G)jbsGD_>-8Dw5A7Eq4pgoW|UNCBhTXc2X* zX(YOM6LPw~Il6!i&tF7L24}rsCm?fqjxE*^>5XtlUSCis0pPDLKWkf|(oGiK5cR1u zf54=(X+wtC1nUZwNXiQ&Ct;S9O!qI$x3pxn36~7IK0QRX6Cy}FZn~UYoGdqsPQtle zwJnb2f0H8SC(0%TBSqF>Q20KB%iTlszJ4cuG^#jsF%(O~sM#Bkv-V1GHb>~=gr9W3 zWM*nx4gJk+CoP`h&5|VOdJ1oAgEq|T`|3{3Fl224g5+yFDms5>@U*823-d1hWML;I z5+80~Xfrq14Dc+We%H&IJ&$X&Bf|vaoF6mcpJ(a>e*f{3iQAT*OEV)|21(R$CelXw zgOL%HR3pJX-YA?kr||+{B7X?-!p{Bf#)UKG*8~>ybtbyEzVD&NNz;JJdIBjX-zf); zERijyI4MqHXQpqhSVPe{Ug8Q3s0Kat-_$lYB|oEI`L0Yg!ZEHcy~0bWR*o@)NI=EI z!7nPKV9gJu*r;?;LE}y=uqFw0(BUPOZMB{Wdb{GkmARen;ZmQgDq*ve_dcEp@mMNE z8wHVC%*!D`MoWh^-F^g<`W{?4ufEe88!DsxRSgFBCb z$N0t{O`VBqEBJ#V!=xjd+c#X;myZFU&9_S93Ifqr4tJ+*IW-_RnB&mU%Lw-B(hKM)3=+F-Sp`ly1MaVx^zaM3??2DLa zZ@7^`EBtyyJ?%Q#x=9-HRh>K6Y=?#Jdb(`TPT>Erb=Gf9|M9!uMt65hOO0-rNVlL! z3ew%3qd`)-Ln#%J?ik&j(%s$7nV;`DzntrPz5amh+TQVu`+nTy3l(Fn+6}nO+tsmn zG-zq#N{R4A79HGNef(1fE8Siy7kB-MCLzUFK%pWRy*&z#DW7bF4-Z(-!fH4E86&M7cnqIMMNh;SalA%t=yAC$?TXNJJL2o_$})=# z=UcKK6y&L^A5HGwq)eu;!Bdk869Q|a$$#_n4Jz~EEmZe2-k$N&rZ`?kEqX7grw{3u z(vB0a*gU{PTJ=fsg*`hlv1*y`%>VI1f8L??8*f32XiRwrC+NPx0xQG7;Q%E9tpSK< z@@vbb&l0pLY0@TpJYBEt0yM_sxNX;h3ShD^CKGTC=j2_k4C zx{aB*CyqI`C>5DPtC1GISP4bQM7(xcR&M>Llzn0?xU7;o{`7i2`Kizz$}jMfZaStQs`GolBxD{@-hm0H z{A*T{%ZNC$FFhhWHJ%AK(m*>Cft@ZdCpCB712DUt>)aq4`dy|!`Dver^`qeOHEcq#Ih+itTv9)%t{QGzL9r750l!4s z%G^Rl=#R3dGj)VgZ1n%5t~JU4urv{0F))#j{XAea#bQFSv`KZS^WF8*P&0TZBY6zw z#(*-t0;NZ3QTYk+`N&0+vXcAzne|(*1nCwl@9xk;Dkc>qxmVo$9KTAYboAn@QFiZO z98Jc)v4-U3qxB&Yb)}+pO~OktDmY}Pfvh^LH->_(mi^;!LOpzJm50CW?jWq!hc^*8 zx5-q*GP7Q&u zP;+fEC3>?r)D>P%F;|Dt2~TFV1afS0DP1;L_r%_I#^wF47G4ax>B(7lOyv;Fb z&jLw37JapzZM)lcxIlHsJ}0jHDgvv+CQygxu;r8yeuMW$IlkQ^izN@GoIzj}`c3v@ z=Qdt5sB~b(`5$uP2eT=Wht;?e4e3IxB>{t+fJ6IPBS)zjkm?X@_b=5K?nprYcyoz`Z z>ldgLLFdq{hFK=2CIF8X99pucFN0Tk@Kc{Du6gacyI_Gk7|P_x z)i+4y!;ld%zdw!qx1`t*z2G#vLYweU z-nrRqCTY$tYmHr(i=!{yhjZsGUk*e*C)9e6yZWz3I{jUExoL}T#bEhVZMVSWLv8#F zH#&TLp*AX?{p7O`hw7InNCos(GbWx*a7pEmB#;klF3w=D+iOUgxr! zOe4}V8j;3P-Nik}S+l7fNOT%c`74MpA#S*k*8FFT`|6&+io{lII7NXL^?=)?$CNn{ zy}P}pz}=(b)W5~+)u~6^53lVNmnCJ~Rd_URpwA4NhV(X1i4h>K2w~pcSASO<9Ngb~ zFF1~GRFquU5HC7iS~ytBYRXsj+=!`MJvEaUi_yhL^}+l`S#+51ZS;_O*5-FLlu6V$ za}61>Ocg@l@7_(7Wk1xWw6Vfqvo4ww#NJ0^jZ_bWBTSLQrbux%D;8H=mEuGAR27rP zSLl=I0;cIMDo;JcC9EBDo;p6tmC)ymB@-NI|k z>L%Ek*WQ5}-1Mh*!SRiTl*^&FKT+7b(#I~U0mJ{UdWBt|nXbmES}QgWPr)lA#g^Ny z*Ll4spHi}m&-uuIzYi|>{z1C>@=xsjY@L7`tgTulZsYV-7;q_pC$V>>c1ffmH1fgb^RFT#+o zzyi`hEDRkB3VGFn8k8@sCpeerpL?TS2+tK z**6z|t9;xoEG$;Cv$MUny;kZBSHM34PptCLk!?~=mA#mhZH;YHKQiLk6%T7V?u%vP8B8=CMd#0ek25S~4oK++Hloe$OT-WpT8gaO;2U9!m zdyo!EmA5Y;nbB!r(f8?MH5!}x$?SvcnODz!X?Jdu#v*)^U0QldTkX}6>HYAKTAlC; zz+%yy6+jO_R_YfCll75oNDHvAgaO1kOB+X|LlGLU?S-N=sSK}0g?97owLi_HAnyZE zFxXGOONA$@inZ1Q=(+Da1dvWx-jI>Pz^G2@sX$%~c7mDPeU2Up)ITd!wPX-qT4>4} z4F8emGn|w*3i*z_HHQEnx$J|wAqKbP4_YY_ZB}+HheUHG*!~+An8f_CxSqsfB&4;r z^rbCtM>7PXYs778Y&dGKBpVgMHQl)-fvS@*_`oO)6aQ4^p*|Uw0+)GxfY{Siks5=K zj3|%3MP=qn-fa>fuirnH@=HkJ;2VW=y%-1msK}&9Ls_&M>hH zJ+^@++Bv(J?Jb|xydgHG@P&lHJ0wrspi#<)(_kF_@hMEx)3zjB$00u~UIiRz^e4O^ z_l_bLNU!^TxP)yA0Dp}-Z_Wu7Q+{&)*Mbzq5Q82!8p0)j{jv+Bhw5BvI707k<23YEgBpHkHR(KU0qX94k5z0$` z0bfndmR`MV_6w%VVETUpQY!`4@7XlpEVuE+>T8L##PV54XJPt!GK(@Biw3!mM1`(szmjmyv~M>uNYBu-gP)VOVGOI@3Pb?c`z zU{FiO_0!D_Dw{Sy5QX6T)klY4SHJ!|PxqYp-Io>$+%6!OJI7rS>SqE5IO%K*VK*8^ zB5udXEuwf`zjp{MADdrAEZmL?Eq_)-@D(C4jn74K!Rne0Peh&xRGoRWpcAL!)pm15 z3?XNjyc*{{z1&JI-5BY`iZ4>&bzFJe7BrQ;a1#v&qQQn|Td!+6$S0{h=5k(VJk|Ww ztQe3;LBlZT+Z@awvw`856Y11T&~j#rFNy=0Hk7m91iG;U#I(=z!VRMyoS+D>U^~Mq zDn{!(W=q%tumymOxD8UDZI@c6Lcz7uJ0a-p*eGK=*L%Nh-W-QncI#kp`S{QSB>@vu z2`-x;UzQ6~du@Jx7&SuB6d3uwWMtHyu}aOp9P7Kq78NA1j}{4%QYEuouro5?@VRYgmRf9z-;+%75%epMrPxxPc7Dh4i&XR+iy zLd&2uh%=wT$L712$@-pcc=x)=@~i+sKM61F1v8R`Y~)XqR=@+P9%8#`<0uY$zh))Q zJ3AiRIZk9%-u{zP@Mv&tl0Zf)Iju#;8+nnT2)1Oa$xl;PVS3JwA=t@3#Er05&`bLP zU!?LKbkOsd>?$^VaimTy2a@!*GF`RP{GyItETyWFZ0PQ*u|aK;4}LC0YwZ!brUjDl zjoa5&!Cc}XqB|#O3$FCV(;j~rR!))``HCiKYJE|kaZN`WLIfMr?d;Yjc zEW&_j#KB4rnByiIT#=6KDn&x(j}60C(#nRu8@lrB@wEXSy*(uqIOhC0If8m6cFpq?}H|I~^6|>T#Ov3iy*hmJ951g9(ZC zT#J(Ho#%Lq8`Ib!OGKV^KV;lT>$u?pE`(K*aHZn`LE6zH8B2%T`HP!qwDuWPYJLoW zTT&IFcJ&>yv9c*8S~04C#j}1mC7!ox(fSt$ziQ6Pb9m&ZuQVnWKPHQNLV<~-U?&5> z%FAuVgGk z0SpzI41}kT)N?@VT7SEdm#>LQqz%a%S9S&vi&ze2nP~Ka9-v`lVF=j*Pzopx$;jHq zW57#YeOV?iWX6x+YuLsh$E+UJ=<+rV3N4CJxGEWwDE1fd7|@bO>oc{{YvL{xa3=|Z zXT1l9KAYkcL~R;ymD^neo+3USS$Gu+N4+`VR9C^4Ls7_sV*Ig1OexN8zVJiLG&MCl zOydnlmqn;9UNFzo(>{NO4$d<>k!p{Xjh&f7<@G=pbzVVM3 zxViqvR#n5?r~Ko&YxhAnvxl+l^PRK?xz)Dwoq(j0z>>hSKR~EH35q3fVPc(@1gOpT>jj||z0$x{ zbONMDoHnA}q@BQ%0rgvA9CE&rlH)WmYzf6JGA{!)5szIb0eMI)CZhQ_qT|>5Bw`di zR>@YAx4^#6BX!FnYU)3;0wYJSU&JuwuJXg|o#P;67yJ=bzk=~*5v;a3=k>*uW}l{da+!daJzS&?Dle4gp&|Nrzk6?|dfSpjlSq|Eb8nUco~haP7Kx1tB7ezD|6cTbgMsC zb5{$pW2%x;a`*Kl&HZgu-YvfkI5(qM{rhLl?4~A}E}+f!;n{Kcs-f%zWAg9a{n?{V zovsk)Q}}~b=b_}m{A%A1W};R)%LypIcqq}3*2?do(o(>0c6;~r6uF~9Z2Ckf)%J4#r+#hDx4qb zm4!Na{PeEyw903HoqaSg%cCj`koPby&g^>LH>)L=S3PBw^4qgs5464PZNT4aGz&5-EQIsv8g?pawNnBnl90)r z%}LQDcu1DRxtT#Cw>e|b%ZIeC*-tJiQJdA7*kzh(o+Kqr`Sw7!^lIadfk20SG&_FG z6%DC{H;qJyWhut%sRO|#xCoXsRb2)Lw-tPY$?jIWV>I)hmr-ugkM?xhZkClPM12}- zXUIk~?@?^hBgL7{v?pbo+#-vW{-lq=YSdG}?~o%04eU(|BOknrkSp7U#5GCI%Dy9) z(t|`vR>M>|pekHYbhio2sp(n-J^(>$%>D!d+Q(uKd+n;*-rNx^ugYimA3w~8o@+Y< z#Sg4a9g2IRoT0;A_o~{~2F@AjMO94tV$J0xM=*&g#)4x*K1 zZ^9|hDm^n!B^UG;ZoGuWIBZr(%Kp|C3cqm??P=M*AsHYlxNzxc-hhAJ6l8W5nwg-f zvC^Zs7wvn>3{{5+A|mgxf>;ex{<1&4UiJY|w#Oo@z8QKAp`~uMsBkt8LO)=cuZ0KK zPdp>IR!AxVZcgL86BdP|$#{Db0FCdkA&jLZ$XMj;G>H$)eh*hFMJ>0+-NBY=0oVc- z2BM4MIGO|E?uT=Z9>46M$#6&uO@IIE1Fd};iSu4L%uQ4vmcA`sOpcLk^_@9DDOH8&`+fx_@Ky{~F~rZnwcgfc5%q@9b@hnI_5#F{ z@v^3YD?KC-0!0y%k$d5JBzqvhFnwxO4CYS)TV*S2R%|4Kv4P_hpzN}2PytLuoIJDV zPe}Ub(Lgd`oYQO{NvV^I{BY`TP}jJ>0c~3KMaLInKSaxeY?3jy)KZ$u6asT|R)*3~ z(&IE=R;X>XvnZ@6<;9)Hu46^*Q*r)Q$P%w-G_NRQ3d3kpf<>eQ;ygUx)uiXMGlIdU z?1{)OFSxsue3t4_Xe=YeR7B*tKPtCn|rGBjhZq#mvI3hbybPxU-{h%TFVj&=@MZwz1bNIblS@f z0S|gM3lmoBFFC&gKZ)PWY-Gz}CCvb6UL&|F)EeQUqJd=*Fc2JTRI5$_ydB+t)*o(1 z1$H_$Oj=GmFv#LDN*?NcJt|;|z@hG5JJedCc)T*8B~Vf<+wP#P_>8B^?cb&P#HW7v zbncN|3tVgSM?V6cG(X*#v@%gGJWHC`^Wf_Mxx?LwrO^rc~O;5NlTpbC35 zl7E*C)CVdTw1e0Q0Ukgi9Y-?9`#lQ=6%`eki*p31o<#(fQl=81K%M0zOL&;pq@E_P zk#Y`5E2Tq%*LjGKwYf*)ulLl!d-3c7ba_+SfgD^~{-&0#^e;UdsKtfY9W2nx2cCJs0C*U?NPCW`OvZKIW7=^BBuV2SEL`vry1ZZqM!`uf*(TRrjajPoV*+T{cn>|*RvG6IVe^oN9?bDQ=uk^maAHw%O!D44SF^1O|9Ho&ql~( z;3tfLPO9qS^beMvlEDJ&OcJ)~5A_V5# zIJD;)ZQ!J&UZ$dyoBE8iMvL*tOa(J%?AV-B`S~3>0BHE^vx*q1c>Y5`1$EV#^9IP< zm;`uDQ-oHkUb4_^Qcbd(eEk*eC%ot>3q)OSWLg)o&WlFA^$g9k2l9=3(a9rtYTV|I zPEO7nLWu3gD$MA_+rLOWlnU>@c_z&4FSv+BL&`Q5A;O-hEE2nZGyEq|VvYvhd}kqp z@A9j{xnQz{B9@qsr21}Qgv-OPNN%q4$SQ~HXbRkL1#!A`MU9)1C*rm)E=`7u`Dt$4 zk!FZnS0~>2Uck$ab!dHgnJs=FPMA#%2j|Sp=F6@7T1nlTyGxE^$wiW^6EDwaC#ru1 zk|%kdUm#ttMhrL->uq$qm>m*SRu|x=ZHdu&y*Xrp6%jE{ zHCO?P3P{2n;3g|RLDBA8U3$_kIvIWWQ!}I6FAsD@`~}jNNwq->L9Rxi{0HF~w!=P@ z6^ueg1t}lWtM3k+wWYSLcqAC2m9sJ}3W5%PrBO0MlZ>SM=51~iIFn*g3sQ*hNqCXO zVE8@7qIvGbB?#pK@sLQQE2>us59Obm3aTn!pzkSH+!qd_bv(WUN}V=11*s4ofvYQU z8y~VX@cj~}kyq6)GNYRKRT5AKs>?nlvp=xdP|u%fV??wA-1Sih4=Bagt9Lm3hMPiw zOji6LONFI;!=OFCIm6yjaRB3!@8iKadqBbS-_DxW-%85y3=n;9Gxx(6tp2Q65g67Zt}Q_TPU&-GuRiMBwsJCE-sz9E=~>-L%uD#nPKJ%w($#AS!6S{L$!tXx zAID;gR&w$C^(aA}?(ymUbo!{KXatg|oPs4Nh5B@0cYiqL;P=RcEH5yqtUN{jL2$HD zOZ<9Gj*JAOa1*YGw{c4o2!qoh_XdlZvvjj@JuZc?*z@8NW9TEr?ipWyE&+xdK7X5+F8zLwvl}T6%nVNYgw}X?Wg$PFZ=ENbA2oG2q zxqry|b2`P@JGFp08y5RI@lb5yU>cFu@?0kG~(+v~%>ZZSfc zZu}xz`;wegxFd%8Fta|XE8nq=@cje4K2}0~=|@A^)t_J$Z;DAj z?U?F1Xe`*Z?Ps3)-WmB8Q%PB|$_?w}UCP6#U!hx~+XzX9bar8@3`-JSDc3H(2PTv- z&C_C!U2LVUvl$55lT#O53g_l#0zl5_BL^|>C)h9RbRH>o?<$JaK%`w=%*x6rkz!a9 zPQWjp2sa2|BrPj$vj!Z>W8!uV{7)43Qs7^mm8XXQA+&Ru7gX7Oc)7-Uz3PFPwISqo zZIz8edIY+-{mE9QY>&IpqMKW4qDGY|cGs1dZfiZdGP_G?cz^lZX%;}L6}ifFOCERy zxPvaI-B-bWdH@o9#R4FfIdWQtLC?cD0YdN1NI6#TO{a5l64b7a-!bp?nxgfl=jD|Z ztl&&Mh<04#dwaCuFMiaX6^aF?cZ%i^>&x9X0WT8JC~&HiRrU4*awTm3`XwuB4stJY z7Wnb;9-DU1)Dh(a7!XPt7bunu=KVluVQySv;dV?8jyUo!yyB6`ss}G#G+L<`*zNeMWmaoP*!AXCyC8;7DXlw&!VJlfMo9ArcslmueW00@{b zri=peuwRwp(5R0A^9ba^5%PHK=B0YWv4b*++#M^LrnUff-1%MliRl5ZGIB^OZ@@&P zt^(i2@+N=$d|x@y`G|_({XXL($6o!6#y;xv`*%@qXYhitk#+ZsScb|c>AD0zbAXz{ zRCVd0^;^fefwb9nzLVSwINGZnu=*G8r^G!6l{y3HHR=hfvLLdt>p&Hf9wQgge+dymkBgjDY z@A-Ck;BPIuw)ShT9J**P?R`3>`YGY=SZ?Y~@qXhc7wl;5WwQt>E5r8gbK3)deL6r= zs!y*ccDWse+t57xao2T0AF)7?>)pI#{~aj71lBjA*Y}!E?w@=)v0{6my^H7=ii80H zwXW2Ai(^)9@R3B7@Co1Tb58B`-w!BG4FNDmQj7--uxU!|ct~s?oTF)qeD5n$r)78l zh37XNp2Ucw6AU(tq{;t{baD?%B^12#jaQGYFIs^F;G_m{`rL?%$P#TnWEA>_iOK{S z3f8uiq-5`1N8T2pwbv6Jd1l`bv8J`oJL4NGp|#%-|KTb3uY&%6CnhgY5wdtU@{W8h zc0*hm%atArP+bmtudOTrAI5rc>vgCAw&{je*O)PSUNIS$bDn+;BU4vHaWb%2BOk-> zH^yAeqDf(r(BxjZDQUd*HI|h4F(wKl895#21zjM`fM6s>q_m>#I`PI{&e3G%08ec| z`MU_tLybc*h^lg%)H~|`#{yXYtmG`w+G~{DfgQ*V){tm|hsq-HJ_UE5i{GgQPd|k` z#(fT&ZMtzj7ruVW%l0~j8XRW>CWE!$L4v*I{ETa<6B+g?%tC$yN=hOUNR_p>E^Vd$ z$@psviq!wHN)Ij&|59~8qbaIfgPVh;gt}_%eanp02y~Sup*ujN0&XWV-y{*eM>NzP za=j?yR?y(u4kW542=Fvddki(NO!Kja8+Up@fq_y6GB;C(Bv8GaE0FHDe|N?Ic3g3= zy72UGJXI@_uzmtjoZrIG0T`#J2Zk?=NUgB0aHKYt+e$HM41brmt3_|7HLvV|Q){DW zjsy?^LZ}pFS0^gmx0Rg0Z$=(2$&@xiFB@IHihCRO3hSL=q0XRB)XzP&YTmSE6+^i4@#EAB!G5Qb$WKg5O+!Z$Pv@nOd z>;ko~@@g!=*9Wz$Al}oC z4*GkfBY6QWRWU>u>VfZCD3D-V)M&fywj6HR3XurdLNJK!WpvrGrhy|w(3VT_j$i43 z^Er+@;(tB6t+?3-(PamEU|LfX~1%;|Uyjbk7J$1AU&f^u}+` z-#osZzXHFBoIp<`3i#N4&cMM*Z9eLsJG{v(d8g1ms{5mI_2QrLu*yYX#kah|37Eo4 zVZ?j0GjbOhjnzp)OKeyIr%*-Sd3Ss~_PbfO@G>oAjsTP~Nwxn0LuhS3k^szLc5}7m zNk58}86qU?y?xzrz{Pd$C;Q#p8}cd!XeZ=?m(U~K?nz^9D{%=uv_^Q2}BJL z(~y1T@?8z4bWCKSB4p@VP9je?02;|iaN$1T`rQ!-FM z;%QTJL9HoJry8TN;3)Fn@brH>V}5+NORTQB7DP((dE@|PV1Zk@s6`o&yWAnNUetcE zeiI(t%LYaT8Ud_{4~Yy7XUW*5ee)zZQ$YcKSATxPL>B4cR-XVQ+2w;`mWiFe#$lW$ zFWg)xNp9wB%CgWdImGv26Lk(N9V^aW9XV7IzG=|$TxRJoDpo;M`GyOz`kTl51PV%Ox%&feum_gDD%}7kFC&4fC#z`-Z&zJjs_STpNexaYsaZ1N3pV3zg95hY}1X? zjAi3&ThtfD%W9wx@rM(6noWpxwW}L&NdT{;TSA_RIGbp_PxTwWzdx6lKF4J17gRDd zIBFSuZSupfxww9N@JsKzeKDN#Lz4_+s~-!SIMk5bwzT+T106wu{&^n7(J{!`{21@C z^ zHEJ6AumZmq>XGb3-40p%X}NzIwB8Ln{B^k)!G0L0n`jFN&`Z4+hFks;soWRel?k{j zO1y(K$`Z~;hLY%XJ(_L^1bQn0x~Sae+UIfRqeGLBjX+MTb$D)g|MyzW;~F>al_}7O`4INl$CJ8{0S8<+i{FHX9(sDD{R@Ek0)9`4QZ+ zJs;i=m=P1~LK(GfKcgiv)=!A6N6=SdyWO_JJ$eg*6m>?*{;5?=>rDaW>%_S6g5qaO zDIJpVZr6PLEtR2uVt4(YdAMW7HYoTx83iBv54XYb@6*vZq>nxrd9U5R!|_M`aFDB| zAC0~vp6BhZNi64msSWEa+?QuzO5S4%&x!oS7^Ny%JaO}5-a^}|RR?`n@x(yuPVq<- z?)SFq2zmG;y*-j{az&Kt-mHhKqKL-*#*rpmsp)Fwz~R7nj|`*v`u@OtTAN*Eam9#y z3^uZP!PVNg*O{uRo}Cpmw7ET$o=D)*i_`nx*izIIdUO0ePGZ0c94~rQJ?U)Bux+Z9 zLtvRq;F$R@-q_CO4s-Yi-RN){H+`|Hl4w$ak2&isBCTcv+@f6Sh_LIr3JVmjuyi(# zC%%rd$~ahR@=!J376vvpz+->;n2i==qx>@TOxHqPoXOPXxwg$!X+}B%$~vpi3g72s zBoCuGR&R=_VX-(HZZ1q|?EK(0x*?RXXwd9afiHN@z2Ux#qk7U7cN(~YI{v9RmiFJ+ z^C|TqG)pAA!PR@>R+v2gypFc|!SG>z)|vd2@Ves7^5V-_zTVE%S>F$YOaeVYW&ztM z*W|~420ae0oVP5_W?u;w#1@l%4u%Cej^N>O@8(9UMBvOujF)M$fg?CuG4#dubdlwR z(r=Qlua8yUAa)0Dzq%U@>ooj?|LTUM?N&xLc#3i~SHN_pR7d!ZdgKRhn&xTaV!B0+ z)e)+tW-N0t*mhyz#1?&>YI#QHSBsRI}f`xS2_Q+&dD9{xZ=|sp=bOj$l8b}##*8j>Gjqz-{Hk~3i4txBd;Y$dnnq^ z4=J6;oeEQgquhymX-&v>aE!gNYG8?|Y><6h`%X-z4F3Q`z>Ju3?l$`=vzaYrXtg~U z-<;=jylRpt0x-1aJv1M)O zZSx}%Fgk=WD2>$KT zhCFpkLy4**pXuRN({Byz^2Ixpi>kDsHUjf710!eaSATE;BR3{F0$%5vn_t7JYgf(n z5j+T{CasaoBSn}(SIzt-FqaIvw6T$@X!T`aQh^3e_j|?6Ty5vh&R;+ybfE7sR_#^A zzNnbCX2RWYwk_**9~@Vj_0QMT4|mTw^pYQiFE7D)u69Mix99?f4L>KrH$N5- z11acd@xci~vI_`JOmMZv{~Y5t0)k7flf;KC%G#q^V9mhlPzRS1wpRYs15twC-=%&N zu59}}{wvTugQDr{F@DON*3h6oSuX0+>2MYsb+3KNs|tmdDRBz79bP`|#T9Mg?CZ6j z^<;)F8EWCqU*q26SOt!k_PdxoO~@$?3bvlbT$VMyw8?9kko0+ts!aQCw`*U zewCnH-!!~%RJb8PQu+n+wO{T}zp&)(OipWy7LS^0nZ3#jN!Rc+x5XPOhN4G!KE0OX z$DN0&nItR^m$zY$d5)yWVlzb&&#aX4c8APL?|VwM8KkWou}Z$BfuktX>&s&5W65`g zuwc(dVX^5ZcZ37|>;rQRhZdiiWc{?q_G&ls)U=&2j_xhJc->M~YTmrO+}&<`(<1d{He#CYfL%T6 zXs%2qm?IU(b>`R~xJ`NqOa!Dkmq(YhO*!PyUqKs(0<<6DWP+3$-pA35X*&kLE}uwv zb#K{(?2S&4dXf?5(-81{NQyGmkVs@DLejoXhr%5+@XZF;^X=aP!(#rh~rkjtgO!opb$4*uRMYO zc9jSH!>9qG?~RmQ{y1l=%SY{ z?Jvm}2dl)@FcT*wN9fTl!voGZ65`1D2loA zUUP2+r8P4oM!+{r20hryrl&hxlSo%N@8>a=6QyT&(y(BD?JHojELo5Fpruo2shd^^ zmid)jDwbW?=#9pFp|kVB+s@+3``%H2RMt*+VsEnd>BTsT>p1iGOnFR+haFLatz~lP zL$941Z5slyTaqmnYM!9L{aC)Fa*D1stI2$Z;gK(elwIC$ZRu8*AF2DwGhN?CTD=!3 z{ww*%TKEB(#0SI;`qpmrm%}}*LabUGZMWTP=#*Y>zX7$J;h+f}-2PKUUlQY^{PvHp zJkf?-{8A=lHhl}q^4ysNScTdMOm6uUeb@Bo}YPz zZmZ=R+mRwF7j2q;p23Lwe35B7E?tV8RnLlZf$LcBQ-Vd{ID=ba=X$JW& z8Z{#6U~HCoy$|Ti+YTqGC&o&%7Lq_ja1iRSeO zNmHK8$77E+-j#u9;287MZ`YPF>0l%;o7$g^s#ix1o5c@Dsin>PKeaghseMo1utg2M z45MTEHB4#POA6|nV1e&}bHEcJ8i(7UxPz{>Y~w=+MVrl|AxNbfw&=ik%(<19q&^Sbpp8*}1S zLYzr7dp@7_AX+qkz^}`0DLlvc1+SfKjHbnxCS^U0hx#5k!C3ot6j=)bGSxf8crp4w z2b=PptfG1GOhRB{c;#|U>3`Ak@(rrlV3zV}KGv!1fh$=wKWkBjEHljxSS0h?UWiQc z{FcHk@TuPo$?}2oVFV_9ancOypp+oPD;?-?u2kbJ*X4HlJcRDMR*tr?+jSAL*87A8 zt*T_p`+3KLJ-D_LcECrFdV!2dYSg+`5s!|vof$M)Q*Ri^FASUtHSGef>P3ywFxLjA z;z=eivPtYjDKK6IyNrvD7AKu@?0D-a>#I<1{!3r1g5|fi&$-PwacmV)(h;PJ8LcF6 ziC~95hr@K)7N&k^VuTNfn*h@}9%s>4t4WEqvCPm~Um61T-5EPvG}LPs;W(xb^4L)% zT0&;BuNB^klpQGR2afb~FkC~GSg8U^Z@%6ni$%PHP=HJ(He5WZlgn(pV*wX3{wm~n{SgO=cxO|!%!nw0pK+9DSOaHpZ#0vy z8mD?R%Vla@F0^0QmS8f7?;m9|>Zw}pu=i(Kl2D@HQ_O2ORk4{GJ^=Ci*vN&$ym$Ps z{(EijSi$IfHrf(&y5c`$1Sj~h7np48#Fi!6Y(-&h+b7jpt?m`)3M}Q&(p1hG2DIG4 z?DS&AcWmcmZ;v@@z?#&BaFy5rP&Q1Tm9LJ#ICV@bWXPuW^EIriFa|BI#<1Ub?Q#lw~8^V_^MFIQlLx9xJfs-j2wbDLVtO%63B)a9uRcEWLOfbb`TXarin`pe9se++0tyzxv?q-G!>ruME=0n9&B4m zE)m?Po+Dx#0TKMs@JCmh1g1;QF#1oF+eLP%RWG1&&Gtk9Ml zYoQ70K6MnrOHnl)c*!q!=iOdoibqnj68!QKd&}kU!&h9TzU;aMW7N#&L&Yc=flcMO zc71sV(b!nRhXvx~U*U#*`;fgbZ z7swVm$K*P~8~HH?UexShZVi}-12$_a1f6;`U`Ma8$j4i$j@tx4HBXC#j$N!eWasV zecd5#%S{Jmg`L%)6Dm2rQ|?N~>`Hoq_gD(8-)Wl>gOIA7@!s;bUm?6H22<^bUjA(! zeRTZAZ+>(^C?>!c)oYpt_?3TZfPqwHs(j3b2bF~^#0W0W1$_tE{=@12R&_J#!NFC> z^gvr7wDhkB=<72I-C{)4k2D^TmYnPXmB09TEL8d5Uine&4+N_Kcj>277A{POHvw09 zxk0;Moa_^O0S8uSe?}{uN42BfTpob1{d<#2#B$pon6rAD{Vtxn`Qi8@ZdLBrB#Ces zcUv+Wn2C2+F!Hx$@rQ(-i(dT~Bcz|+3Por6df2h4lV_m)Vmg4EU9#@-yRvkD*}WT9 zzve~W%Ln<~)REcA&;#q`rc<$%B6$29shuXv^nWk4GA+Q984=#69;SijWw<)&CHMi6 zA*l1s@IO6AEi$F@*B>tKK}IKV%7pO?09y*UDs7|`S_ucy%&o8^+ozRH3HXTt z=8c1J2LB~p+|9NvE6K)D8=U@bfK(9toWlX26JWL-+}ebOHo-|NWYBQK9#>4DuO^%a z*V$g_0|*RAN(<64bT`u7ozit~p69&p@0@ksvt}`i#UK1}&)oa| z?!B+;`s`f>TB5AU!!F(h7ir7iOs}PJ6JxjCEK=0ac z@%)`dLW?93p%Z_IX%+v?b|XK)#8 zK6$oNst5ZKXp2z&rg z2bdi+Uux8uzaXo}tFF%h*Ax4w$2^(byV~zAn1GIdDTohg-Na+u@o8Q8W93+MEh6@M&_Mu2#=llEK4ay#>xw+?q>d_pn2glemY{0FwNhym>>Rl87HK=Pmv&0HEozZ zU-$mx*MFYrN$N1+yf5mcvw|eVu1n%}Z697U?|Co?)7U`ITKXCQSOe?tr^fVt-2MwH z2(?+8rwN)9E!wA0^~ zCamI4v#$?-_!y%`K{;pHH?1DBCWAH@5g>jEeYo$UA10FuI%UwMUS_0{R#g-ob)#VJ zj%NfHOJ}MCB`)rX%LD{u@~AHkCAvy=z(J}?{=$L3gn_!%d`JwGNmcjJ*Gx<0 zLF!`y$GiEnqBVxJ)2Hz`={dp@FjLeQOb<R)^htvijZ5U@t zSwQUC>dSJm>`U2|^9~?toHl#x=gR8nMD~xU`xIBl$C}D5$4?nM1Z>{_ino|oJ}zGM zKED3vS~$y7$h%PJxk4G==&G@CQfQhB_ZloouFTUGX^X`2iBM7zIqe2bTf@Cfw@l&* zXqV(&mWU^U&Z1p|YnT0Z5(+m48QRCSY>gXJk{ED;T+&ZQmKTV%=OxYc@XSaD1H0lg zd$qodgeT#a&mgzc0+#wJ{8b47OfmJ}53ki#Ue;8VkSs%-$0g%7lyoEP z%Bo5LaA+3De$;h(cFaW-E6Z$pS2L<4CgR#Nsp0@}aomp9Y+!K_RaRr~BcwA=iGNNM_iIKBw%DVq_h2*hVblHDkjV1vc?}ra1pROX z{UpwOH;0NGl|(O%6hZ#pVlomG=;WlwJ=4_y_>f)4E#kT!;7GRUR5m1DWv=#$*w1%; zfA07wF0Kyfg?BH2`KQVE)wL5|=^CP|GJ1wd_aAbf+_%Z!8oAckmKp*9n`L|HT>1t! zNA0A&FS^vD%#O@68edMDDxwNftM)2q*3qoUVw7Fc@u#_Ke7Zz{zExFKsRaA@2h3(i zTA3M*W4&!2m~VGu8*!{YM-DqFCTVm;P^i34_@i9xgEdy}pdz5Ynf}5}T~)R6n<&l1 zng&t=z3cb=<~JciQPUB?>)JX{kTVO3clh_D2LDtMcm1uXkyOJf`9TmWYgWs)(cJsv zS$BVIq>_xo9EMg^>C%u_cIUQ8XYcu=W zQexOP!EI*XG9#4n%dsbEPUMV^3iS9zx}jC4Kp+7)+{$#Zf_qK!I# zw;KJYJf!#b=|Hdw1_LJFf`qak3=Bjt7x-_;7lSi*0#Pao2ET}=+e#e5B9;_`XQDe z6_HHQQGI8kfR?@{ULNbgcvBpi0~kA=P=IJ!NkE}bpP%N!U^1hyqK}g{_9MR+OqZoY zWrWaY8S>R;dg7Z{gw+`%&0!1D+Ew2$+qg7QFJ*P%>DJs_~)s% zOq<{q~qkqqdOZ?Wuesu7n-G_5P2zctA}$)S-Nd~ zosbzo__pC@zT=I!Xa!ty%i)|%DK?en4mb9lRTPUtx`jd}-uH(gVxK;1o(Et9+}eK8 z>s>=!3Pg$m1Wesm@NyJUt-}W!A~uPMl+PDedB`t1!@@3go^+MDqE zTvXZs4y~wXlpuMZedVoCBa;Y{L!K2QppRt5l}y)`4GUMtE2Q}GKZLG?j4&UU+Y6jZB5B ziLGy#6oKvWx}M1F_E|4%fJy>td>S`!WMnAvRmQZ)=0ghc18KkQTv@_RT^k@~dJbz~ zRnm?~fB9qyKmm}ZM^cqS=o#FnHSQOwuglAf%rxE^J1Om9_fWY{XneKG#mDoeem6{I=w1on;Dnga z0BCci^fX}rTKI$vZr1Qf{V%-;NWKWpx^eb6Iv|O9kp_v|=pLHTv&i~>!yKNMcfs)S z0~XW{+{@8);{Z)P3<*3C@RbsNllJTsbZ-JzsWfYW%R&Kjd`>bBE46tqJA(1aNY?SO za8t~JlF(#o96ES$?Rg6&DF^NcCQL3UdfVDOlVW}23CV0gaUYMH+VHO@Xf5*lpsC0p z94dkPdTr{9K*JcAp9k9`A-x=Fawfb}d9ZC?Hin5=Jf!+rsFEmImn#FxLOK*O$?&Mp z;9QzRj z41jOKM#?BZRD6QI*2gQZSx@c;%+W?U>}bq#&?U4!Eq)l0#$E+y5W zq0U8Pv1T)6_pLsiZHYrE1o2j zf=E`BQx@SSXWWAzLf2`<-{=2F6Udv1;la18(C+z_(##gG9@s^9yC1&++M*eThgn{D zZ{(~cCs=!i0ke1Z0#@c+QnP;k>Roy=VHkOOEy%ufp|Nr`1{=NrB(P^~2Hc`IMscOf zAfQS~uig#YsL9-24*D`w%e>ZU=C_+!gdYL9=P~W~w)Q98I4i}nqIMw!v^FS8q8+d` zXna6$<0*7rlZ5NeoD6UAfK*qq3sAoN*v<9;jV7Q=6$Iqtqqr@v7)kx~F%lpH>*5nG zFZ5oHLQuUv@zAp4M_hgRebvf#Vhy0`Mg>YH9K_KafO(ApcEK$$o$XiGhyC*N%G*D| z|85td1cpTKln8v8s9K=5+6`G&AOs`K^ydVA;pWPPdpMPN+|yC z;`-kb`#1knVKqiB=6u0-0r7_c z?b9WIHBG&-Gb`8C>6`uzoSa3QohbD}ZUm;yp7ShpCYr>Zje@{wZWy3yC?-B{DlrIm zkraeKgok0Z$<&M}cCTcG#qn6Zp}NW1|6rT?3>T%!;_5@i#JT3=!U3XI{C7UsPbQPQzH#dRE)-AXnD)wo=$th+xG(?h z*YQ{#ZvhVwIjIP8n{Kx|K$=x<4MkfiMn}{su@wD)+3L^L}5Vh z@PB%Z{Q@jkUAGy=_+ThiRp8(`gGb8K-{LRN;@KjdjO#8nq#- z#w2f>Wrg#B`1SUI2ChFI3&k&9ojFzvX42D}Ze0wi`PxVp=vUy(=di9}$|wdvO;GCsuv3qZn41Qc?Uo+@t4fR=TpG0 zxxXD(74I(=DR%V<2oZdW_gZ8dVHIqjf4!VsZ8&N76ED)cH*31Y=d@te_E;ohte`$Y zVuz4h4a8WSR3#}jXnDm)Bp8%NIP;?P0UNas;mp|{Lr^`@@ zJt=9>*=o?b-5vmMrW!o~=t^Z0rHhYpuDDJs2)*!7hJd`s$3weH846F}Dg_cD^>dc1 zYadKqczvtnuqUA0>zbf$v*s?44S3P1z78ACf#f-V_kquF<7-l{h!CPz1iZdHzEah6 zTjV$SkcfWiB*m%xYZN5psk}Q5w3y1E-46u8{__C-#Tai0s^BSVhVZpR(;!C88!GOiYk zWaEtq+^G&#G8Y11o}*vtn_xsK(jB!m6|Av zwQYc!vs$-s8O7Z-Iy_-MSX_VHvkHKuKGcfU+(KlSnd`5RCxua#RY zYI@xi$6!=^fP0lz9UoqfB`;13Mv|rM})w)82e1U&YYP7po zl{+Kq8p36}UvD&DKWIfOmhN+vZymDRe*aQQsrAkpt#3JiNTtj0M2fC^4_X9>aqPpk@ziK6z!>#6){S3kM9= z9dNm-0K_adF<#f8&sq5@ungu6HW8?4S7c?fA-OvYnR$F;_TW}j{RZmQUsYMKhzDpv zJT*%>DTVAv{JM)(Vz?((h1#1-G`R%5Z}35QE|pDnhAzFiX2Xkz82PcNe%9TeML2Vu zB=}?IKgYb6=Cc5r#XA49`X5zOsmk-EPwn6Q?{nt=OFdi!6uTV0$M@2BS-f_!uy2;d zHPdGXLI=B=pkE6;1TX2H+==BXrY2j}cQbVZ!0)tw2NQ!gd^VZ)SnFpO-M7Qve6pNe zDX8#x3_-a5Xd*y!fPn>|M(coSf`Bd#Z2=lD^a6!>0AzA80umh6V==#KT`h|jC+uMh z+L_JJ$;TI)vG+Hf#T$hpFn$;leRp z^L0-L+a)3MKZ#y#KDQJb*}|^G(n^5qajY?=m%fDAeLDoEa5FA|>=gLAECvwpliHFm zMi{n+;0gGazu7JIK2B`9X`hu`45;+Izw@z};@i^~6zfg*ISwmp=`Ni0ULrLe$&y7+ zrQLhyKgd6*+=tfqV6=o4!Bm-Zm4fj_Z`PXa8;G z=QN+It=0V8Gz&}qqnrOO1|Hsnhf$F2ALLUQj+29@WsEWeyA+wcD?6p%lZ^0ANLC`9 z1n~&};o3cEMx#Xz1}U;1s1_ZE!M=ub)OJCxhn7P6JCJag@B>KWH*mXYjF(=DP9PU~ zlB!$V(7A#map0o~I}@7w`))Ksvl>>Wt3wh5e5wozOndZt>=9b1_lQ9*#-9w>Lf3Dd zqtKCrWH@7DC>acA0EX)I8vx2PgkZy-BR<6?P%dhDeMR*jbnP_*=Msz4h+0)(BS);T zk?sp9i5me*a18Y-CRtNo2G>@=P9Pxf98Q}`5W5gRu4amr>e9gi)TR7zHc=aMM>iAr z3FsK!Q|*qb1rf&Sxnz7e1Gq3<}&7m)FlG2aq+XDuRgOGY^ zK&;-K_&4DTn;L=ovn-vLTubjtd6@np?5=|m&Ds0mwb&9rPDn=^sFBwGC_(ivmhL>Z z^!pK%o57TNEs>(Q-juUZT-dx)cX>yCsrWrWHbzM%JrL5H(mm01ce>imoX)0BHstc@ zu_Y)`HRI;)yh#aQ44EI1n0ZR*Llz9#v%Upw)wc34T-kc9P^~*WD)>{`1&GS6#;wl3 z9)3`~I0ILY4&LqZoJura%vQR8*@d#)uZ}87x?M0_ev=_sY7ro|e3JF$GLF?&m2-0E-6is zf=97Zt790hbVTxq)u0`Q5%Z$+8QU!K#3E%coFDk5cqsa3NOrszI4H_{Fw4)r2@p`S ziU@w>1&qDJM1cngZ;Ep;Yfc=u+_ma|!qcnfT(lg3{V=7v#i_h=(R?hLFcoUDl$wR_ zvTYQEbcL6Xjp{pKNAt^p@)w9w67j>z|CEpXb=0l7hD3mo7>M_Brf~u~!53l{$Z}pn z@mYo%AUrennM5$ToC`oHsF3!nLjXNU;jUqGgX*MH}>j#@n-F;Q~v}}NxuO_;-&rb6{Rv#Hl4R znc#2F;_1;UWV8RzvbWO`#KaLBiNE;K9}LA{sBY?ctj_bWcz>VFb{+}h)ez1vdZ~pB6tX5mUbU)`}iH||GZ|DiA$%Cqi zaMg#k5|A5$j-v7D7h*=#iv06#D@HGB$Rz-#nzYskw$ke*w5^K+QawIzwjJMwM;2oq1|8m=cr39*h}oiQ`tT>Pd2ad-+zZ~QJbLT!YSIe<5!kNu^GCyvulywY)^xCYrv%x5?;j`1-sdC~303#)Dq|5P*kG8RQ|MPQx^Bb+_YG6w8EKUK32HTR-VYsa~4~_;hUKR$3?F z8`40d7B>O%o^;qJp(n1Yr!oAR=f{x?Hksf)Fnq!6baOQI2+;xmO7U69<-$)`W;aR- zGqh)mSvcyTN#{f~h#JMmrGR3GdSF@#)eB2F5o0Bn+dU%ty06~sQMs8+zkH?x1#eP7zhl?s!GWxx@N8n6wh(()R zh@C88IWEtAc4xk^lt|)u@VDK@Fax;G@t>`9zBbGul=3U~IYx%pdx109ownv3(-5 zZ7BP`cMvdwp`ZFB9xprd1dYs2-o{By-~U#FJFEb`t@T82;Q{W5v0Rx;V~6>6HkOUw zw`d}}csK{>k)pb-0T*4K>7RA7!Em6F%=?BxPpM2R(xIJ+QCoqrIlIP-L~k=D>_vuY zEaj5qLCJ92)#-Jva<;W!V_et0%p(TNlWS+Ng0vA@;h_27s|wF|ytZh*Q`c(27Qgz{ z{Q8EgEkQ>{`j{AJt64|s1HOk|DL$f~F4m)SBp!4B)C15_RJ(%ef@h`|&5^S&wCA>n*-O zDOWa#;gQF`G4z7hLd1>{V;J`paP& z2_O>H7})SpATGn4`dgQoKQCD$>_D^1Kcy(>QX0kXe1V|TKczknIReE`?uum5;7b+i zVv-BJ#V;=q{SFN(#r0@iI}1&|;YR-&extFb-iL*0@4C6+@1Zxd>wrcOY&k#J;@>wM z=l}DF&-!UYWXJ23cvxhW@J5_EL&6IfhkC?<#?y^i>tx2Q6KFm*s__o7 zYd{*xwXzH1peBRyX|ufU%_y^w>YT?D(uUQd{m4=@xO)nzA-i4AG#cXuWI+>Uj63xO z++7UOcz3;eDbt=kbuXFp0eATurIVwcy&`ZUU-eD1)zvrO8$*c1?sLDNpmZQDj4vGl z;(+sjh|jEh0?MtfUny~@@2WSV+ADkK2sQ-W zt+0N^Nm=)Zi*gFEhV@A^!yYwm$Z_BA242MGgBx+ApV|lhrB$y5<9H& zua_7U=A*jUpl`qj_rQEqGj&a@@9@r}*t~fT?C%+95p)hEQ}lw4_|v={;_$BLxj#|N zO=|s{VCv4j>u$RJ3@FYT%1BbyLZ4P{>>=**q7C z31V}9D@^{-r4&E7n(KZEn9%SIIrGApEyQ8kep2wbhwkB((>c=&Uem-~Ppqqa3%2AR$dI$`WF(jk~YpclV_~x1Fe_%g1bU$h*n*mu&mg;v$ z3e1fest=Z%49Figyq4;tDDkY%wI4wi|K%_l7pKG$p9wMQz#XMLBVVLK&Tc!%=`B!# zf8FE2HD{^mp24U{1)m~vnm#Xa56Iq|>LP$`&Qc>6-a;hXk(b6BRNWrG*t)-q0!VD1 zwv=+zoCQK1ww-ly7R3EoF+s$7S zly(3xOv6|reUWBrOvrPxrhX2uogTl^Qg3@|2*Q0OQlQd~6)m*REcF(;GQM=Vk$z4; zULjQi3cBFqVB*@hmlnT&oa3Be1fI~?dO1}Eun-u>aLByaZ2$aK?XI?8(X2i=T+!HR zZ+pCPt@X|8_i>3c?cgpZ&^)moTBTT5a$^WsyaklR_kJ|0sh&tjVLM#wr9Zb zRG;n3$l^Z8^$C0%J}4h^xQoTq$|7T>kb#EK%pC@WNgwfjk>7a)QfaQNF_W74)_nAI6Z3*1Ts@sT3@8PtHZPd@snsjHy+;91hl>ur^X}mU7OTtPp==czNEV|;_ zEqcA5E1Pa?)%W(&PFl)thiz#-YMg;++*UsDR^6J>`jPY34fz>~x{ebzz6?kT<(cs5 z3Q%M}k7qA8KD5>NV?burtBCY(a}Bx#`7)k3K5 z$7!voRSK02!iZW#2##AXp-*@wfbNG>d7n$>s3BzA_iE+svsZQ!)_-kNqN<=ZV-lno z1(MMiGEEYVSiXhLvOl3TT&Pa-MQ|$^*)nxp_+YTZ!~-?HQg#7wej4x`V5r#Cz1W;W zW!4L;$az*d(|}iZO|1Gu?VuprWXLbTW7>)s zr2&zaZd`7p5pR}PWpkcEFWyKF_B5%%b%E7DMWduDtb;cX+wAQk|7vkv&P(!z!j|kj zDmk{Yz*}9zNIt8`!C3&wTa8*=zkNiJ5EMI83g(JzTByW2B@3o6&L!Ly^091XIfDm- z{6d~Ogn~EPi(LUIHJMXfe6SCN9x=>^d#UE2>HZFV0v}|KW}qpr_hhi~DGnJZLKGQZ z7iwlHnx?BYQ5rF)$=0{?FvPHXf_m)e>wnPv0dw)6C!NTkzpEbr#1UJpE&)dec*~2d zH5eG6Wi-;-p}=Q~f`m9jfT_BKf{GeMcuotDn+<4Gw$z%x;mVMv4rK9*0*8fT-!jbF9tzLJrHcx|pMx>XV*!e#I}WVtTCygO^+G;Z zaoMJ2As;~5{e>I%wAa9ELw@=AAiK$??N4rhc_!!UC09JDE`WWj+pDa*(I8+|Z6m!7 zPwn)klNTboh&J^P$L+ZisM*!`QvD^YyGgw{=Q*|7czqy1dsN|jA&G%RkKVYB;S2sr zjRwMC>-Ys0r_I{X4s`fLzW^Li@w`eHtmTOx>U-lueZSroSO@C+8t)>Eh?MeAUgbSVL%PIhQ{=>J(M2t1yA2gMi z0C+lQXGu^OxZb75k<6u*f@0^Z-8_3rg!^_VuR{ulL%unG&US!#h0%eFTf7X)-z))Xm0Nu)X+^ zsi}=0v4kJ`g4_tmbcNHmzT!|V6F$vw1VO>Vt+m@5LGtDunAOeh@*J|DZ$wPAS7$mC z)BJZQ)}RXJP%>xAC1Iww+MBLobaXKeDigyE=dFOh!Q?l(>K~Fb`7sA&#VlKg6XQ$wZ8G~h1J!ejw24% z#o9=vl(Kuu5rvu7)ac@DuxLFfZeP`JfQnjJXsIvl z%}SPKd~NN>ka_OUF67rJnt-C*`-Rc&6iVKr418)_C7~3A4m~d2Cd+eN+^qrUxcirN zW(2qTLyF!fwV^d+;B&oTR9rHz`A5I+V`eXtf8z55NFx}}6u$;Fqsjd&|3!|EIrUT690f@G`qpEIs(X3o?S!CxH0MHI9{lLG_g%Gz6_A53o0lnU)pV z#4HsM^LhbzxWe?4)Qilo3_I%hWkDBc1?+M|?%deokp||Vr~x%W zP|&=$Yp}*vskJlz>+WVb7`rJ4Mu_zZ5R78GR0}5TKL|oAo#m+Cq`$svKAZDCpPNB% zQ%EeJxgWl#Q>Zt=?i*qo7d$7hLnmK@e{yRq_hME`vyxX53ue?PJ76f(V$%`@lvUk~ z^SKT_U&|!rRmV=IC#P;x7E|3ij z6%`H%BHp@L+U|%|Y0@!AyNtBw)!kS@9^R#C_i_%FF*Z9F^Pmb}mA#dvU##l=p!Eb| z(D`6C75UV&oyJ-BBe#dI+b!Q=_LV7xW5k0*aGO#OWbGq>PC1;Eu>w!(S2%w69wHegwL}`Jvork>bl+C|z>x{uXE%b^f?Iv`^UR?U! z=(bE}_$VT{n&sb?EyLH)K$wO@a}3eJLj8KF=R(?~%@Y^&;k=xehFq28Mv%TCno_6u zbGSq42c{lr!56kmX-=H8#JDeY>)BWChM^lH#S~rrRm=oj-0qJ;#otu@x?zM^=+oo! z;D`lBG87coYA(4{-&Y*s;ZGsB8o7RkXoR4C8D&z~Pe3{Ax%$0imz74yW2Y|4NGlJMjtxBE!r zm68(Y{C&?$0o0zLLF^Y6HOdQ5XFqXoxcaWfaUahh1N^o=guvY2NP~N+Gm6?$-W4`r zfU~dIa+dSu(kQ+x!5d$GnUaoO9<63bbjbnKA#gx&C2@!yo^?xCQoa1BQY_rh64TOiDIM{ z_p*ADa=RBz6-pLH$?_8{1aX4o?KI}Bbi&|#IMk*kCu5$z1nb`XoR__Eru{`AJtk^> z+Q8o?i#^HcJg%fB-!-5Kzx!*`s9RoDX$>za@XxoYS2N_4-w7m5HPg=k*`ivh-()qr zFf8Kf_RYxfKoP5Gn;?e|V&1{DT!zt;A{BM>?pBEqC%v566{@dR-((G*MW^i#&az;! zeSOR~^n@3l{Ji+arE$5R~ zZ!)ErnvN!0meZcjlQule{aSt%U8erff4te_|J2N%+nvUfdf`JIMh9w9d0BhpC9W7p zNUwTBQSkwv4s6|Vv`_c-as=B+#FVf1-==)d2gT6S=+X;+JHyvEuuIm?TKouK16JUr zNKN6vp*2}Ij^-}%SUZB!9sPb)`q}Gj2WrSDZ(%@18P}XX+sWRv^8^w5Z_D6M8wU>2 z&Ua}&mA^zvMvgx@ICmJ<<5YxQEE*sGU3-|wVdRTKaz+PUH8(rjrFW-sG@N1k#h(lc zdJF$2xC{th;KdDq6&gbz(ebiam|yaDzARY7i^Lf$GaJr;v?uEs*r$mfZ)}VE zMnnbl`Z4$6jrdqRr5OP|$w4R8mFRBYu}M;M))I6%wi>Js{B?!SxN&G)^0tdOF&}UK zIBFzTp!5ygBuaOoA!MxIcOAcvLTBaXdMoq-cc4|$zHW{eK2O%8L;_lUJe(}fqsPpa z_YYM%_voWUhy0^gbOuMR0q23P9n6nXCUnh4jBFrq6f7L>;pMA-L0V>)&Q)^by_VAA zqSkm%`H56%E0c1ODxl;o#GczO_{IBcS)H8QESkE*f!{-_Bl~11tx8WbDvR2oh7=ir zGxoo(Wr)~ZEIfWiiH31ZQ$+O9^!-t*&YHWFm zHrDElwnqsmh-#GDJc$Y2T3I7@&6?c?SwNQYqrVX9mgBt!%Mti#tkdo1Cih8wEfeAX zi&<#chpLT}S6g)==?3H7SMf`k#zQgsP%%Cm==7bQtWj)Rku^M@ua!0g6 z>lHO>LVy2RM3VelV2d};k_7GJju(am+J%C3DcAbPoxKOee0E~uHA%4NT}Oag@7EI8 zSFJ8$4c$4KKcYL3t;|PfyM7*s$QybLo_n^hn4#_dBVIvlB*+$f*?s6-Qa#}FVpSS{ ztSB>*y`_9Z*em$wDBy*f(bk_7zg@Hd!$rJ($1raQI`8rGU$~QtPpwETi z0j>iNorss2ugce-zY8E>g%eOv+mL?4*qR6ja#-2$4)uNIQkhB9BlJ>ggE-YNy2dz} z@UZwx3p3-ECGb=ssn13C$M}S9`ifOr8*Hwpi&ILjpXPoU#okCg9tiX^-$pz6ZlX8; zp44og*SLYX3MWqnTiNmx%*?Jg}gthd=&31mgtw-xPsr~kYj}U2n$t)RD_Uy{9 z>4AEef)=9#q*dGk*is1Pltu(=E+uXk6$+53BsbWDZb+&!Xl|)zK)-1Py&KEVR`h`3 zVGQEVK4sg^`*@_-rw1YiXdt1-=u!}q%<|7wNpzBWr6)A6+*>>+`?L?TYbX2kauq)c zOo{0*`CUk?HW)6YX*lW#9n*RZ-P~O)wqW>qG!5e{@G*%a*qW_k#3`H+e4Q<>=D{HMJ;S-ui(O5WIhT4%}!26L7g1Vf$vnpwO62517N-LO;H z@5{bPL0xXLq|;GI<3c%zmi`@7%hhvX<09baCo|?j?q6-a?5Si&GcDP8of~D1L3Yz< zCN}tF6Vgk3dx!_x6gRuxbcLQR1vr`(0in>YKJ#-BVnW({ymxjPXw}$@WId!3m0X7| zp_6Obfi6Ct8V9(KE%8jQcrU5eG(fLHh=(g?Wb&_k&!$Wn1!lSO?F3IIUPKol7T;r) zq9bd6aPfGTGQgK z9@addgtLd{ZFvK-tsV~E7T`2Hn5dkwNsPdT&xP~azoq@^>UX{7&X>P%Tb_Pf7|qDG z0fZb4B8s7W#i&Qnp%dfFPiv{!2i(Rz?4%drbS`iNFnNv_&J0W93~$HT?5zX$#v7v5 zWc6R=bgcagJLD1mLd+vvmD?`Qbn(>w1^M8~o@~?4FI+wS>d9yHBv(T4eC+@q`nNTD z+a8Nt)9+zg#S#tT50H{Ew8x zQ2g@a)S898@@Y9Ktv&9>^XhRFiy z9;k;APhrX39wuBVb7XH~bt;JOY#?@yO`DLHbTFf0+t4Ac8FdHFCfN=v<4F zzJlh9^T5*05eijn0r=xfE`QF6U3$Dy{;(S*ESO+WpzumV(Kogv|L*?mNY;=vxf^1S zw24bA0t4hQH)7|c0TSTelYc$*J;QMg*Vgb4V6`!|FX9krszgUts^6a77Vj0_{Siq{K;#DIyi#FFl~9<1T?LH&k0 z_opAn%O}%#eU%6&{bu3V0UOOtx|Y(6pAxy-1=TS_^1kdQwp;Zp_uIU%TO(wL5onPy z05bUtP99#_%CqDF&qrG>?MDAgP)YHrlr|iXiGrLBl<>;T6wGA|gT_YivRQdH5G?R4 znDp1J$1>GJ$3!r~K@@lMHNfXGV4uKgiD1HYkzf*yAi|J|OZxXPulkhUl;m|wrK_ZB ztJ%OM*y1f2Mv27-Yg&tu<0up7YCIudRG!FBXTx&-ezk>U2Lkq3cUbK zY%fj9zPV1$;5mJ3bXNq?3A3sPaLGy#mQ_BE6AM2E{opQZ3*;7E0gIcRAk=@oGlOoi z@@#!BRDf6^#a0(=;qz=64Z-3FjJe+4-?_o-aIhQMzB{n3nxUc_jGduYB!vV z$pQ(@^mYlhM{o`j4Sl_f=molX^TFtPBJzrV%7_T*Dy~g+(97_oB+oDK=?ip|qs+{(034xk7pvxwd-loD84qj<{w~L z)AoIUDNdu`jgn<9v@*P9EPj$_TOkOazoYTgcrAt0@Gk)Z$mE9#Vo@>yUj`npJdJLQ z%Nk^jAOH6GBaM>QPI4G27Y$SXkcbC1`rPqs%SKBe4gS>; zfu-+)I5C_3>a0qE#Wj?<64aO^*$uJR-x_)Ssteg#6Ft9zwRlV-^d%3o;a?w@t|77y z8RSQg3l`e3cISY%z{}a>lL=P?&cAyorRUZ~dcY&Y2{k|lgjbvQ=aXE-pO_e-b-QIvm3)uAHsNr`9 zzsWXyzG((4VC8(v`CpHQW>VPo6K%6Hjo zY@T+kxVYgZJ?V-*7;r$Y(zPYoDrEScH6kxD&b_^AU-cD6(IXNT!aYVUy2o%{p=GEs zf3e9z?p;*wt)i#lYS(ufRTg`8y}s*HLE|GZujnBP`U==%rGttUnr!Aw8!80MM+IjX zxE4iKg42zO)W$>!c54wJ?GfEZgBpGDX|M7P`7TC1S~eLRfWQLt$@y`QMDj1FHNL}r z4I#B}iZh<4CHH2?wS^#ZEGLTM>(5vJ+CFKNX-OAZV4pS}1J`BY4kt+YpqVh>O6qpd zp)y*P`4Fl0=CmC~_6nFete0c3RC0P-LN!q0j>OELg7@L-XH5E}C}R=AV9|i3VQ_WU zN}h^Wl_zyaDIoj0a!6}v{PodVKbd?fZ4tm6MGj)>?g$z8Y-QuX7A>wSnIHs z4U|HA+nPj=6RT9(aJF~?NP(IX8_w1t8|V_ag*!TV`d6beRqOXaaY}Z-+>BeOx)cT= zG*31692cyjK~>_F@3_q5bdI?DeCe^`@d#afgF;*6rImBq0O8+oT?SgzOnwuj^MVqg@earEz}%4uHlj9SGkZ`7USROYSDs+T>fEmvMA ze2=zsqceTpSEm|y;MnnJ!ZdOA+nnt|$j@D$|BI@#Y-_8HzI=jHq=Gw?V#OUokd_uH z6fN!!#hsvOp|}_K7Ax-VQe27!f;$8Y4nc=!uK&z6^9jz2eeQkEz4uzb6`){rWpN~n z>=C$jm)}_V-VAVJubry4nX+UwK$FJdNg_<1Sz2!mjw+$4QXkrdRslA;_poZ{DhWL> zXz2WH+}_ZRBc^9_?)gQfC``^@)?;>-xZAy#O(T@cfnw4#kA2vcI~}tqW~0Fc_42d= zc}$?%vedpttS$A6H|!-7Tj8AjM)lFIoWbGxY%a5!D`6Qb1(aNis3+tHYW71+~eD=uEQ zw(Bu*0!#pzLB-Pe1DyrAPoVs$chW}C@u+63y>IS|TM{fQPkLM4Zx=5(LN}iO3$Jin zoFz-PzlpBQ$W~Xm?l2vB@eMP)RPL!nOfhJ~mAe#PK^K`SNQnT{FjQm*JN>$1?mt@z z6j+1%opj`zG2t6u!hco~VPQV%&_>^&RGD7xfCt~F`JMada9m8W1SPC`6GIrg;Z;6+cTV}j6Z>F8 zx|f(LX!hk+MS&8>t9XjPe?%f>Y>8hPsmC2Wtn%GZ8+%XkCEjRP8h<;bTUgzOL3{rIir7Wzf!>!JZq;(ZSx^dwlc~#S!krdwgS#|*okw2TR`R2^k-`BdZx_`$BXEH_rO9_ZExS-^XR@9D8ZW zvFzZ3j>9p?BTp_49Dm-k77ldPLx^$D3CUjAKR;jp zy@nA8MU>${dSom>I8idUt`Z8*p+pl!l?!e>2hG^ilj!y#GpRDZWmBE?G?t)QOTCJP zU?1(CMZ75IehD9f%VxxzOx0N2&zvA_=*!VGLH?=0#rY>!!5kNb4XvPef_|IJOy#)o zDqxw9godVUXb*$S`r8g*-1^}TbBH?IM0qI-v+?|ELh>iiD`B}3TtMv^C{fb-(U5NnArpzG+YFS3N1Fd1^c{;(5|`~o`f+# zT~FhI&rf07>=%E&Ccb^;LNmp_v=cMx;dIO^)I=i4rJHHLP%k+C(}T>g2Z+VS{*-5< zhb4y}O4ZfM1h#&(BRuxyNZF4_9X>O?d`{WadswJmzTv8wd}csxl-BrY7mYPe`(Go# zaHUT{|KFby9&CqYY8;H-i_bQnR(-xhZhm-YS~T13!^)z}(Q?QLzDSW${!Tg6o*!dU zxHZc*nV0EpnaCt?_4mYgHo(p^f-~!u$ zohQ0O0z$Wnh;xw(+a@eML+!Lv#grr~Du>}lI5T*?`|u%jRL`O7!psA)k7eg~0#&Vl z_%Ka0?q4qmWTEc|$oSaS8?XCg{H*{Ri4?Yj8twgQkqz zPoimFy)td>KJzE|Ky>6?%_4H`CEi2^;h2()i~5#0R%@HZOYmWYU>Pg5Ar9riU#iMN zbT$QKn~&ykg?Q8-4#Zz#uZV1Zw@&`&k54$Vg0IZ&Mesl6`!4$8e=7Rmx*GhwQ893L z?nL}hibV(%GE<@@?D>Il8uWvgKoe8io@KcSK9X4c7Jmx24;~#n_sOCgpQN+49jZvuQiewX z9B>;}|COl}5LNrm-KxvQkIP;@F(Mk3B+&^;TA$xhGQ+4n1_FPUJKHx>28fSNug#^J} z{+^V!B#-gvk+A_U8`D`>ejDP>hQckT$>RdU#%?wS+4TbcNJrl>(jAU(qu0FqI^+F? zHlQ>Kg5~wyG8TU@-sfqqi(aOu3_b};4kR&}C0eL_oL^-Gg?NoeBj0IYjZP%?)d&an zwo*a*rm+i_SzzB@4R=PBga2NlFgN-G2mzHWla3m%{T#Fsk30D{A*Vg`AO9KpKGDUu z{xc7~!aE5d=^_sNzt#J{=kZ~Xx{-mNYZaG}T(eA<>4~pX4lbgR=wbaA%t9j<*~gmc zk>s(2W@NWAWGU9O&GlN!08a`|m^cBB*p*t45mg$Oc)WA9S#E7|LrnhlGQPjwh#_zl z{dY$$?2uui#_PVAyq2Fpm?`*%6tG}|zHuJ1QOaTn_0R@aRyY9wXNYM`Q5weup(on->_Shak_ivI*B5(z zU3oQ!or}MtQ6&hFfa(=E^?WZli-W`v7}{UFs=b=m)RfaauC{ zn{i+Obh|B%i4wcuXOiDn9w=&iXUt}BM&?g0k&*?&fir2fX z#LQvk7L~%yrJS~03Ing{g z+3Vc5SsKd5qnE8<&(jX4U zb-ZAsG|CZgdD6sk2~8F9CD3X2->*Jr(yY9Y=Z{KfwN_HAo63@$@n6z*9$L%etvo() za0rn{0QxFL0h|+K=#5)V7ci@^C%g_=x4H7WUV}qEZMml$rA0npBE<`mQPqr@6R?>N z{?1yeYjy^mfnotBpI{aur_K9Fc6?Lw~ z$aggl<6fxBGy^vI$1m$}ATE~Nzm?#%=qB1V#q=wOQnvgl&lx8cKj{-7N8W8k@vAE**((&@UD*XwX3 zF=S*U=qqSyBlc^M@>59Zz~d?aJw!*D^$(R6i^7TMb?DW3m03S!(ra%x{_=4Yz^)I@ z5KY)rK{h???sETN>?|6*LEf9eZxv0U56!g0=0_Dc6If5Kk0ijk0rkGQ1A162PXkPo z_9E-q)59i1MsX}D5$=_bV?8y>&#du6OWX&2YXmr~8w?5OJToX#CYy2iXF(EXt7)KV zMQj$mT5E=ta|98_o@1LVb?f^7|M~tdx+;cUduDntG*eDlkyWIJ2VbC{QsT@C7~HG5Ssh1nBLBSu<4>0`2|A)yFQ+6K?FJ)i0;eD$q%c$epb znl6dkNJ4g?!zb}h(^4k|E!m`)>|l`pba*=z;HH9ylrIb)7C zR~^`6IhtlMgl)0;G;hOIw=9#7;d}2Lh(&Ok_xRMbLh5X$`fAE3yG@Fj4&a359}ACy z*`PE9P>&RkKRpS!q#l8=r;*CTBPV8BUR*_r_PCH zv*qFNJX$mX850#4r=y-2SMW9;&y=xU9Q9z|6cCBPnys~<|38(q@- zIR))cf!*#%V&#*ja!&u}SJMy#Jrre$PbUFZWD-%N3ULhm^gf79fmhM}L(lv7?-et5 zg8qBq+*Y|2t;SOLM88g3pB-+ARlD!Qcr4h`ZRuOB#m`>=ngjo6B2^WmjFJ0q*6PcehuY0QW?1nU=WAu(I@4u_A@q{f!&LO_9m17J0oG zN5(_1<)%l>g#c5}Qirq283hyBaoX@1Ufz%GPAL7ka>fO-@XPGEfYZB^O7;NR`vUzo z4?#O*m1NuL>NT9vgk_B8;+aBdR7X$wqhGPnVx?=c(rsI7gcetjGx8hxmLmcNF-iDn zRBt(3UCDVndo#a9A|r9zQZO7~^i0feElBH*s;aB)q_iyVD!Cl%f$O(~QKQQ;Ep5fH zt;uXRVYb8zrqLQQWt1Oji1H)Jv1?Sk69AZg^+;Q9V`LrxhD!U&h`MY`wQlOYUqOER z>EmY@V294e*q&u!Hx;IE7gXvA?!T^Ow0hoW<>IU~j4Jg?J{@hHA;1rm;A)(%w_gB@ zq@T6F3JMd`qp_~DgHQ6z8v~6sku!>G>M-bxNsS#E(clHLq@w-RA$jxr$F_7KZsSyn zCfww+XZDIz!HD!HhTKyIuKu%|ib^kzo`0!gdC9sI)jn58u2{DQA%5YDKLoQ#)Xh!Y z-1cFk7=@bk_DLK%cEsN%LM>k#uJYy)@EnSyF+H04;Ct5(Fl2xsLwTyL*Y`vC?JGRT z2N0L-5rjNlJ<&)}BYAX1ibJCvoUey=YOErMMA1+W#(BWh^J@0aO(Hj4=P1x-`#*8; z*ohl`uX$8OW#gs}eSfK8+r0JI&Ce5@H&Km!yyJKtsS@0gp5x8#`xuNapYm^Qr z0*P4H_M$g|CbM$+DD#IC2;Ve2b3l-7SCp3G*gQlS+iWWz+x0}Senf=}wcbla(eTTV za_O>WyFZR(1Xh30FHd&VwTtPJx850}8~FWEGWGoHtiS%zKwOWn*g~DH)uwLBx?z(G z(;1hllm?&q0R8@Kg>b2{7?lh*518Lf@xAl4u-zOFMKQ91*Lxhc#(X79^=)T>TU%>A zwo3Sfp;iIxuDQ>7`pow#1uo#UN{_3NN@OX-JN(u2Rm+S$l{`o4` zkbK1K*4E5?w*08+1+~vzg-7;X8ntMhQM&^l=cl}_1yXWYhfjZ-XoSKIF~w3>)}ciJ zN9@st?oclw)tec68}YRV&FzZpx3|&v2*(E8fU5Oso-n?-&9OqSN7)cyw zsf#V5@$l}>VE41~V_kS&8YeMoDH`Zkew6`JdH}Uc4{hpETRe_Q>uoOm8XwwMzD&6D zBo3=yq2SqQsG3CfUzuN1vA=Omvl}zu+iy6Tw7O~GTQ5sgy@Y&>+=+@_k>oClw~D2W z>Pt)5xs!Nz^Kcy<3Gr#mQ%nN&)wGE=)+exkk=50Br0;8dc#WU3@R4>%OCx;S{~!C% z?q5HP)9Uug7L?s!5|>(E#Z(Hj%jG&Q#XsLHEngvTB}Wztb*rQX+pEZGW#HE)-;3X@ z0YSyQlFI9eBD|^}5)qb6}!y4GVmzJ{Wwn(@O zqJXX~Uk%Oz{C>D3#Si_UjbTfrYL;Anrw>wb3rY!&W-$kOkw+MyhHJ4zTc>GbE*2|W2$RL+@#&m5y!C| z&`k2F1MKItc{C>{XfK*CuU44CzRxHwc&z-aHVz_etkzHr$+Wo<`4chbv=}ICV?mN` z{V|+42@UfZxoFtqJ-2W1=^5y+7My#6iO3?mdUfWTv25Yv835ZNZlo0{QU25oA=KEw0+B+CJ|jA_g}D`OX!sNGnMRRQAYn zUXCvT)SJh0FwmKvenck=9qITFDOq^&<$zEO&IcEOami1K?Cy`@2#f(ka3U#d&>s$o+vf$o#?bO1#`pY>YPC5+4gJsM ztY>%OO_saZoZAi$5_V1PTZyGHada)|f?4cZllO&VqFRdW zPZjZ7+IUgdYf*b|4<6RGp}SRbb&9C0>yxT#sZ6Ke=`r9)%+exmIad1Y7rp_1Ww}-| zY*IXOC9KLyxBG?cKVs!${~?m|#S97xN&$wapAkCTX#lf03?2jVM`Qb~WJ!1`3p}2M zj3Wy&mz(F8E*&0kn%DmgIVzO+Ynbo+Ds$0--5IJ%+(~@dH@+5*qA@X<#?e~* z+}|?n)uZyKB|QVs>^+QVwvEr`uS!KuWKqq~*nK(%{Tsa-xmS2`L&Sf!fdmaSi}WQ8 z9I@0o?)0wV!YW400q&`n`zoBCui5Lze+srFd1dB0m1B>()_;>Mr)Bn$h5p zL5#z;=9(=o(VpxReHbfwf8$$U{51cRI0H}sRv=?NpOWRv%unVKh9-7*+!K#) z+{35iCG1J%v-wI6B&tU@0}87?|L&~=>Rmm2UFM(o3+Zr>luu(#WsT6-zLQY(pBIpi z&c0Fb7EEQp6H-)7%_`Bo6dFE6xD2dfuA#5YT=-&rVKbS{zPn|A45)=DIw zxOYep&32aBxXMMvj}LEd-RF%3{+hS^e863dTL>HdvwhKm$?@{{*edJ3Pqag#5p4X0aA-PM=h@leI)Ej1A>cAZ zCcIQL?D*`5rmx;S{vS58q|4+R3!||3hl7>=VO-1^@q*`JUZ57K+blBiuCYWWDwcH5 zkq*jUOb#GVko*s3vP{t_RK%`VI$@I#--HRC?ue){vjx20!7hS(mKe>U>!;M>hKlbmO5q3pf;Cxn3JLIL z(jSGBu#X6$ZR=OW(K(jXBc=D~ZPvPfyx+;H{`6o9#o6>kooD_Jdf4slS zYHaTYi{J~!#ww`4?(A~QWA`4W#D08W0)z0SaU!%dMh&2G5Wy2`WTs}VCFMw9NaQJah zjIpPyfl&D>&W-Kvr_RW`*^`QC@x;w?4ovg@U!#17$?xKkVcnA-yIL0FnS~WwW7-4c zgKG_5d*+RkZ0HCfjYyc3nqJ2Yhq;D~j5pOd9jVd2@Pwib!fQ5Rr+1+!39%JS+XwyE zR3043A1bh6hua?lY22Vf)FpFPS(b&@3w`DvUq#6{#Gn{#;Dzfb9u7{5+sIiZ_YyMu~rf zqs8F3`#s#qGb@cF?AwmgJO6z`4lU;{Y~B74-@H6u?wp5k>XC70T|pb63f%rpDj>fV zxpoMopp}1J^e5dV+9j<@$J>auv;3vlW{-yHifPUF?DdITLzP-}lsdJ};T#K;hFQZ) zw|{`2708O&2!_PJ|+kBKTxghy_tf`Sv%RU)SbZE}DKlQ}>W8gY>#wRqjUqD!|W zgtFus81K-{JiA`}DWGGT`mmgBx$ORyoS{LIdF-h}z}L<1Skx{&-1)96w=j7$*%CBgr5RG%jw53%G^nU%B= zkQs)cQ=rrxQ6^gr@dh0l6zU=4joWi}2!4Q|7V%4$|z8L{?0up!33Swey6Sv1XR zYwh!=aubCa?^I!_RcN96_1+d~`z`K)iiVR$9ny$e|JqkrmYNP}?1G=fxkKDnf#?Ti zDy7l-jlP zB`wC`cH{I#(0xyU0&JZl;4nGOqhTBkfKH(Ps4}*$p;!KyZ~g7v-1jTF4V=O*Lf<_| zF!tU$7VV2Jbr zVpD90bmoKH_fpu-lfLyx%rFa}a&Ni|%&>V70x4;U>>6;sl(PNjeqF|{`#TA6MQj8D=nK~^{jEe@*O(7=wOwAN{HN}98)(F;YyIe zZVO#V7k1=J-z;%g%WrnrHYDZLH9w{19-~%B6L1@$58UQfNbz^8hS2g139(IugDG?p4Y!kMyl+}|?|1nb z-u!4XdaLkf@z6igVFnldI?1uj@4kLRimNeIUO*ads`c!TB9ryMf2~TJDo{1u)y;FNoe4m7A*Fl|SRtfTuOpkj9%rx^H{o z{F@2lLw2LFLN$@rHQnq*Ivp|UG8YS3JkTcL{qv+>c+CTFyuv(%o;A=RxSCF}SL;R- z_AYEE&mq+^YvbSTN1*DzsX;3j*Ud&sD($-) zYVV(-Andj-F>koi4@L88$NX1!1f*IodwOoeAV}77=thlqRy8nrXW{%Fne}Z=QL_2s zy4ceZzlc*dnlAF_OSNe(^2X&V^9mwgPyC#X)2Jv&K`yBktJkl}^p+K}EinP>fDPKB z)4F1(ANupJ#C5rf-7Z?8$GteQLW1E6*Q8_}C6Qs1j+)mE7O)@B8CzkT3w4&g+Q+!SSQ4$pFYt&LSWqO~UiH zL^Sea`42<{w5k%T3CoSsjIUE?u9pGCNo!ebQP*t>u#;y0DKe26!Zr)9vHIEMx|3!D z--Q+bq;;j?iEpHxi_p{}6?!#$^AzlLjp39ztJnjMGYkO=)}HxlG$2|}e4f}&qw(O3 zCup_Ey9RFJH(11W%Xb!C3%88LVyPT9eCYADDxC>;UB|5D(b@PaMPnxTfhQkl)X+0F zQ(wcON?UpIM1ld~&vIBn&3ZzPq|nrU+7gTPSQxoQ3>s)GU6@<5AK+B7fe&wTEle#Q zck?UgRL&5Y;JEZ#-zwaOCJ&4fOOe%}n4I`%LuQe!gtWN-IQf0CDm%j)~f z{YvB1_=s|j<~AkEvY1`NclGU}=zc#$uVfn>!~(}e20j9|%csHLK0^4vr={1~G0VU5 z>v>vnn%s;4j~-!s4qYrs&?f8cVsvqI&`G(s%nCnGx}CVkwt~}=C!zi{FvkPL6Z{}x z(UhZP(|{V`ugobc3z^0<+8XoR>-rY}!uB%UE<~#B;jb_)ii)0=?>``kv?M9eRXqJk~`bC+qHAF>L0eGjD`8+1iX%o&oWhaGGs18-OAp) z+^MoiyBf(?h@+8ghx1HP{87qBK5V{45{1SwBVSLx=7tv#Qs83R>n(;}owobkDz@~@ z*5Gf_qJ{MsgZQn;E4QeSb{iNjzP^8dEjC_kR|wl?%x-cCdc!`NGJBBp*EzJ7(6e$~^Jv|sMfYehh;G$uD zx>Pgu=yFG~HL>3MC^>Rptz1+Vx{@-z#|J9fKCI^##fzJU-%LGk-@FP{e+KoJxFE36 z*kDxleLXedx3-U+uRF1mr}|@7ygxu}2i;;aKQ8OnT28W6NO|Wn%Kc0~lXAJAf}@FV z=M2IPZA^4)uC`E^4-DqI?k)G`WGD3=D&%@Y|F@cepI~D*iXlcGgqd<%pRoh%YLaEq zOFZ+tTsKGm4#|`fHuk-5G_BkmRP7Xo?n`vmiF+=Yqijaa(Fyx7R{|Ly9u`4sW-K!dbG@NUW>@zVGHjC`8F0bk=h)LWWHnlfsz!Z#vWs( zp6rb(x=E$!%m#*$L&sd^;`c9G2&7wQH-+;|f6c2Yrf#~zl;ZucJwG)h_obLc1SP~1_UtJ@ zEc{8wE~FA%1u8D$rdbVViI(?6E@k#nrz^=8>($GrUa8EqmE$s$9d+e!3$`Ej2bqm(zy0hdP>{77YxA4sSK42o{a0E~l)J!}W)H_U%Aq zl>fx6g9Uv@nXxlN_U$@9j|4J#|82@y7u3xl^U6u7&L)$-+?5Wxxadw~-aceRtbrR> z_(_rj4;GUg4N`yoF?{p1vR#>-#D8*{X~+4bjh61>DE}?;mgbJ)QNhu5(0%wrlFyE~ zxk{U}tu=&wFRupgG(G3`Op_mRnRW3kYTZ@mgM|()$8{*-v`qD#eeW%@=q;wpb(X5> z#vQ~wmqVN8B7M+*p6Jr2*=ggs%r;|cbxBKzQhmtqykzoVWz$lgRSnyGU)USK){t8= zOLRx#&$rHT{OX1kOCy?11(B%p=vIbqSu<&i>2%wXRel#q`!9|y{k5~3EwBzRn(SDr zS%US~nEG}zlQBNT3 zsN+t+#T_OFkoEi`QEvjW@{)hs4Vjdy!|D`!@2-pvbQZUXk3 zF1Hal6=fxfE9!PP$@;^LlS|cic$5NBd9;n6sYHEo;!FMpr+2q>o!lYet3tPaE|EfN z2wT*H01bbqJJhGsa56fo%Pqt}F--|ehu)@B#)^^`VoQ}ty%X~>=8}8Xwl0~?iuA=* zzny22$(3N&>jrdjz%IVAHXQe2(;7;!N7_j+o>l$I;NiG*;>pg`OR%kG3HS{t3n8TW ziiHbEW5vTC!Qi~1&dF%KHf6+Q<)Z>%6bM1So`Qcrq&EM#cyZhXYuQyNIBKJCg@SAY zlz3**F4v6J2rxl-sfb7X+GE2lnuFSFa^DX|nVu|I|KPPY>NLISY_=LP8K*KdGD3g* z-VJ99g)I@Dr&tb+$v$4`SnTo71@Zz0jadOK^ZVq8o^5Ola9Q9O5sQ+_@zuLCal)Bz zyFwu7Abr&t)mOsEZ92q8ZPyZs2s&w zLC8*fA%A(Xt&S99gPo2e&j@Yui{XWM??8$SM%T0hHSJWV%I}1)3_xQk6Dj64+o73!TMnSwB@N49#D1#8=f$8 zdY*Ty4*yn?uO~4dzS=sA20X|-tj_ll9j0O-a@|Ceelg|SLByk#&x*#IVse*cmugR) z-jrFX;gi#@{DKmCHB_gNrEaITZU3Op1nv}41`m207tqWdTIMEqyUlh)gj zJ$g*!+bzu1b*Iv=YlZl%pEfBvqLM}^YUE#Hs!w2#CGeI2o#6d$u7z$mCRKuk=7S3> zGwSj`R$AkwBzo-I46~T=L5kYTM$eY+V|R3|%2JKpA~}ug9j;zc2&@FO4J0ungGqmh zrYXXA;e0`j$`)X()Ju@S(*FZES!-uj1&3ZomNa-WioIo_j5ddS{UXp`l=LKrNp!fl) z&3GFR()bFgU)&G=i;y3va#7o|xUpTrpFS;e?HU68Gv$=nGjMAzix3J5h$(Ha5wGyU zz4cw>UXhy(UMcyapj2}k*o+=@b-9-r0iF;(Ff2H{?{&!r-B){9^?^^4Q? zen*c}KgM34?{&Hnc2c@trQRu2(h+k$O3`UUj@{a;bPf}}<&R^qV6M0>9HSFnK+vqo z4P}UXYSteaAt^A}eH=L{KrIoUL=H~V3^o$A!Hys7!+O^-#5@jorj~)=Lt=jb5y-4B zEZ8I7H+AN_!htmo9C53(UDl!=qK|H;StwfQyun%=>w(WX_X!NhpF>0QDhNi2IIq9S zb;i!cDnhg6&BTV1oXvDWAwhP%b5$FW+iXusDfT|mDM?tn|5}Bv3d;EL6ww4G~sfUM2?lS6+&fRcB;BvNux0G z4yOuZ*r@R7jiDh2W~*Jvz}9lzx*_~$m2@gp^`vPHmy~`qJNPXfA8>OuBBw+FS^$%b za+7IcP(XIT10-b@LBSsEeBfqNm!bfPCZ#Nv4biS0^z+fM9m%Hac+ z(H~*!=ji&ztr1?#fCVrM`OPM7qysgqiuMnMF)!MPAYtw6P8q!SRoVN`aB+FZ15+`S z?zg4J&y2zv0v`Q)c+9ZS#_4#v2!8_})F1yuOb^brmprmVyx5luF_uSKYsm|`bNoYH6nW|#iF~kK&k_eT0wkoA~yuMNxoR{ppxuGm-tUdA@H0AB{dko zoxTOu@-0XAcI6hS{mg2c$NxPR&Ddj_2eZ&-;oo-CM4uQPDSJ@hwZH&2K6u{aqi?}y zJzm8HH)GFM9|k=@kSF3a>{d=S(Pn$pCNl}YR={0_Qu?l_xOcE>6@T6tC@9&lgF z>hg??E8`@5SSAxYeOCabWU0&DDKZt<;bZ?JPT|SwqDN7-m^`&F0ui!YOGYEYKXw?Z z=-Fz>luWHfs`NWnfF~#R!6KNt*Lv4J_K#KelX;Gq5M-*OcqwPaKsiCyN{E?}+Ulm` zO!sH@{X|T%czF8#Cg{UE;Tf&*t3TkWx=O2AB8&7Q*uguvvr}9BVNc;|JHDsPa@iQZ$%H;_qh^90iFP;k=$A0f{unHZuM>9P@3HVE^6KWzGygu= zKhAr|lz^*t`3MTL>80((+x7zYz0WO(y1eBX)TW`b_K_lHUVOEfSd4AKtp11nPVOOJ z|K*y;?k!i!dPNT#xc{yMvIs%t!I>z=zjX#})R|m<`H^UW`P?K(qF9fA;V8zrh%Q=s z?|8&U?n^Ih{mei4{?{pm|0ck+nG5S9N2s#lFOM5-gm6t8ekPejt zFKDTI+GC{7BVcqlpHD0L+o2+#LrNQj2r}cN0+Q%Q>m>jaCmIj@jU&gAh6oA)6^@3rYk1 z$T41s0|7LikiL|;Q$ky&YC^RRai^D5sTpSpWs8nPTm5#piAGCNwyh12g%Us;0h@pm z@_n~ZUkZ-(`er0HrzGs(mxr|wI@1}|hAH+#Vk7NHhd>5OvcR~{D8gw#K2W9!`&@A% z#GT3*OASwn^eNt?RAWvuqvpwzlf~A@%e%0IsEG>9d&+V3_L#seFncWc%EmZ>miyp6 znn1!y2TLWL%FcYn3_U4QFXWv?TbLiiXRkbRza2T(q=V8xIRAP)6Df;{ zD@n#J*e!m7V)3P6yh49?~B5>~uOC1MXM8onP^ga!?YX_+G>-P-x z3r;5cDMW^wC+hw8)n4@2>v zQ^AR#p-=fa<$U%UdyM&eH3hAlbrj?#dv1y_MC!xazg0`nO9RU2^U8A(!u$Q+mjG`? z1V;1Y=8p|N7Za@}Ze@A+`A)@lWrFvTwC?>IkmHym-{Vvpx-M6|z)Rg}9>=LXMV4JD zv*Zf>#0cH!Lm@ku6#o72_aTrubmRF9wa5Hr;64UcE_Vw#?||{uul}&8$NX$yUWb=+ z>!4)IV^`};BHLrDz+*u?aE_BG?VkIBtpZd=qC?^qyHieM0rR^T^Eyx_x*KYk@jVnW znF$wq5x~c1z22_T5;9vgT#vDARg2v>dmq2NpzO;)M%QNi^HSHJ2lOOTqJ&%_mJQcD z7%edVVUbJvzSAPQtNCi*-)Ffi^;uawM+uRnN{*nIT4JD^rCR52ieahcK`I5@`q2`< z^vd0*S#|G7-jTbFBiKr(IJB56QK+kDik~+)^)g^qw6|M!WE5ZX#VhbQCNS4h2h?wh zlZZR}mD$aml|&j4M_0|=UkueIeEqmLRQY3RbL{fjHb+tOVnlu)c9Y$g@ktuk z{#Mb@9On@nY4}vs)=O3QoQ0<|VY1%5N!;%6N+QK<3gzC9mixnj#(DlYNMGhPv`Pdo z>ifwC%YexY!w)typ#B@;b)rspqgS0T2$VrxO&Sd_N)9seZ-AuxI_XepJ~zd(t~5YG zR1~nb`)14cV+831=-hiekQxyA4j`@#nt&iKK~*sX-1J)6c$3ebQV9JER3^I^#{x$G zNy`v^-;r(lwPAYp?D8x9fu3hpmz(L3Kb@b4V$qw=0Kb4KgbBiMl8_ZpEjUyr3CE=B zZ+k;CG8Ec)wK8D8i32c6O~T`UMnOC*;%XQ|75dA+n{T+p4n9lLnI9cbNJ><*l*+X) z(iAw8&KM$m`{Zuo!_qgZIHgp=CAvDPTJrd=BGdgrQU#z2RhcP~G=+*l&CatqS7{eq z@2EDWoIj{j0TOIrq@7!0CUXtb(tg5A)_|_eS^W39FUT|*)C6Gu4=AgmQt#Q%?v?RH z$IgQ1We~ce$QauFPcd zn$%zfAkecS^ZKt|XUYxt)~(UgV1p?FX7l&ZNhMC&2v^X+zJ;b!R9M*kVRdZoIf zr=^Z4s`23k%KW_oLgPlHsfF-kc@&tu$%r0Gj!!LW^y^)tT5;@_Y#1mqj9V>(j zy{Z?)Wu#AvaH1@z(*;r0t#00=<9n)B(zKF3A8s83Y(W}{`RTH{#f9fq!^LMb_f6k? zl%Ud*H_Q0JozXqUS^*2*aj>fo&nA~G-@J%t+a-`f!77`Bot3h%O)F`XWqAIDcHFP( zqTf{e?;Nh#UtP%x^GU^wYGGN$_x&#bC_&f0!8svm`m^sAeqjLTS0C*rE&4Xe%R_A3(eC0JEFR_8}_WUn_kN+*TQFkdIh>REbHF}tgT@};UBH8;ZZ!@1wRo+rXtAuLkt(CPiMeCq;blHcu) z8t^vxP5Yy%&RpmRLFzP6W zFe!XVR@gkU&TNoAK(96~M%e?9rYy!*@*pnf3*;E;9eGex!D*QIR?@(Ith3C8goK!~ z&eY$oUAvm;sIYlUs8mFt%yyf;t&iy8+PclYJMPGCq9hb6%m9uGJaS+BERYMUiD$&X z3HcJV@ErJ{^9P36%Al2J5Jl^rk=ls>aBr5p^2dPWTk@`INFuL;UbvMFitBZ!OPkK? zP$UID|GbBh)}ib$xbbf`>Wu;8bg7ldu-qAV?@bu$%=*C4cqbo_5wqI4{_fcv;P_|Ch_(}xXqu#J z9j2nk%S;yM_u6q-f~>y!=w}9$*u4LD1v2Z(my_R;kw1Md->4UCrfL7Ob(T4AdIjvP zF$Eplu=iI3;@WUu%Le6JNquF^+K#8MI5CC6runAn40-Fjx-xoUPorOngDAIUR<_s(Fi_1roQ{KZOd7$iElg zn)iz6+5p^nbLf*gjrFj+|J)D*cH1Zbd7YmfDCm-o|LGf*IM^Va>h|~O?ZV#eG^Vj^ z*s)w5xVMVD_tZD`P_x9D8Z7I1I$gQXkL{#M@3InFwUdV-ICQ(x`Um|vDI8@5GJkgs3;`nd$lD;xkB{5E(KB45L#H!6&Lmtxf zuUxyV?A!h7fbp_|roP`(&Qo$<;}(CZ_jKIo0~gFyMkPS%$~zC#r)K`apz=LV+ zyrZYQyWNu{dX&EG#?#0YRii%`Qpov+Q5llAjY6b@(71$ z#q(CyL6ij^5O>+uc}C|KfEPnU+^HkJV8e9Tb6}f%_`(Qf>z4C-I%XKU!L-#o=Bq=t zqw-R(GRk~@NE&8-O$QdPNvnZ1z0$dG;k?;K`*p9%dN6w+*X?<|;s7KD`b~+={o0eP z+4YNE^;(`$<~B0}{Oz||)j2Zc7JgS1TtDlW67Q9mZlq5US?=~7-6^{cZ<2@a_0A9X zG1s#lANB6Ux#Kc^>DN9l&E%x|-y|9;vyLvXNoJ@!IX^=IU8K1?F$aDk%j2bCG3419;^5il|r@&f0QkYGE zwlr|9bCJ%SJ9}7pG~~N??=I7)>u5>@yD_@!ZQJ$@)gE0Lg((}|dGIXF6nhM(C`^%K zx|-XV<`2V!wBo}IkgFXyo(~Olru@CETB(4I6)IK?h-qMBo(zB(m@CKg`H`g@x+$dZY4rs zE;?Dw3=Z*5C|2D6bcyTDAL<$9$XSM96Ectl0EO0dy?VZ8gT-aqJj7AI2F zN%MZP-w9o^eX|)Ly)Q%d07H}A0w=!W1rlbv{h%8MtI1HM;tOOalAS@#njW!PsXVi4)p|42+QM}aJDo>tpkmKmlBL*-9z_msjk?W!z`pPRURPR2!`M%L< z<=yVjV$x{mI3z=?l#+EoOE`mpXw&n4vG(MXPntHWb)sg&OX#ct+732fqq}Ecpfm5d zYY}CIyrR6(&Tgp$37mS~Xk$v`v*p^#cXL0LqsI>^0Qj-d+dB~9u*S*iv+s%>2i693 znb!m0+7qnY{*yYr*<`$v13m)W;Rwuai#Gl!y{3i(MDBfT_fjy}IxKSTuTbz&`8vgI zhB;)N6kWV>R!(0C?`Ye&XD*yHtH+eB33L9DM=NRWo^J?!Y zQ}ZtA_v5?D%G#i8sZz!PQHK%r<;d}aQnqj{)y|D<&pMA~9riEa(wR&xboYERs5K;wymQ$%UIWv!VZ(s{7ErwTw>gk1Gc%3;UypU18Ev zS+lhwoLAZS&gF6i_dcEQvIzoUn4nIY{MnJ?d(Cz3RuJ5uJ^sLp99I*|r^I)jHn-U9*XQweJFz>ff@|<2L zaHQMnmTn<$4}MC1-SM-zr<6Jf#1GqcARO>twS7^%0&+7~50ziH&vRdj*$z{wi9l`N z(XCRYaJZyM)gsTxN26X;nUPm?ns-JSrNH#lWnT4})Rw>2O3%r!YFX1ki{UT$jAH`} z-CbIGG#YC+4Jb#sy`p*4qGm@OC{y1p`+k#bnR3Z9Wmozoq1GU)RFD&$Tht`iad4Kz7ez;3ZOJrMr!TM>2{qHZa$sIR)IrwwO!x84$i>FQS z8GUbG)q#wI%E zI`^^8^P`z$XCB!7=w08TgDTNUHQ2cEy^J-8%M`w`W5<||79S-8-fE|u4n!bEW0h&x z{G2vy*f8N|4JHVk8n?knpn(fp0_|Ef^_)hd^ytx}vVHq@<8X@eSwm8xtab=KS0+Os z_VhDPYa_q9+gLmQ13LNu`&epD7XF4d+#fj}c8t7q3B6ZIIvH{KZsTb$mAt9a) zv#)fS(oRPbCq&pDJ*Jcb<^F&TUuoP7VX&G9k_)wK*H(ahrOJ>T@_?SOb*WQs!`T;|KA6<7a-7U@G#BZZ z!k`J!bI>uwLpkTSW*P0Uk*xU&LzaL+2kyl>7qs!6%OgjQm`x`$p1MdU8#Xd}p6JwC zbF=8t-s5@DnGVsiNC$iE+VIX;Ow4whrd@LX95YH8Xa^4-^jtUR_@ZY|uQJQ?<9!gH zMrYDlmUs`s}NI3Er_7yw%Ez zxtUo0!=V-LaE!rafJ)wv(P^aeE`>46^X7SyPn4Co<`w-)FXCqwP-l z@l7tj*Gm}d|3Jy7F2+EL}BhUtJmgCQ7gF(kjpCwv5yDpg7b z=)gKM0x*MdK<8|S@{J}R7-Z%Bg+Wx{TgG5`w{AQAGJs>!Xa_)EpzjgdCJguizS|jO zfYn@^+xW(Ppw|I$fq_qKf--jPz-1u!>`v2^l?V`*`o{esxy8BuskUB$J0k~Ocd8Q~ zERDkpw7mzWYGWyVOhvbG;%f=ZXwbawGI4QVsZhL*YC?W4{bqk+z-EA~H+9?DIYe(W zQKH1st&w-fn9hC2?^Mby0>E=$A3$?Rw!*Sa!54`D>+s6$&olSSUsrM`;deW+GGlRE z<+kKH)fnUYyF53{tlNKhp`W#Cgid9xV}m@)vlmZE`ZSs4!}{OF$QlOFE0e#PVI`rr z+jU$MBlY&mV;{?tC13QU>2C!#6ClHwCH-Xh{7+TmcZoR;t=#FWzE?o+khE0xu+tK2 z{`&;(wbPvt)?AsL*qpT!W$BjbW*q=`_L`BBFGmTvGuvH0!5_YLP7AFV&Qe>lWw!jd z(z!>UkiOfGojWYAt2VRSzSe;OXBmV&m(#_Dw8#AM*YDbP_*< z(TL0a0>aMva8LNjUH@NG?e7#i;Pa{KHDfDx{3HeS8{=e`x(*Q}efqUp;1#@W{|5Q+ z`*V!sBg+fr$_{GHQs8dpSs)8dVe6`tE6}`fonz*}kGs95Kp>h!d1foep%;g%#eX9TP`rhv%8h5AwM+Rbzr3eZ{w zbVpc0bSeQY8C)PwpMAEjVX=@VATQ^&j=W%!(+LNVphHS$3+HG2IMzuP%R;(y}`-P_qiyg)wq(~9NBsCY~_{X(#y*S{*aLhy}(mHx#YUlFh%MB<5*u{O*>2F9y>*%`WxZz2nnp_#o!pR+5Hr=5_<-Paet%lD$Q({5yh@cS zemmhNQY`-QzN2hmycYp4ki3AoHFer_Q$B9wJ24iQo}f|5i`lbhoBUv)i##w|=?d17 zUpM=^1n`u4k1;R;xCU4|Y0@MwkTnqIT_73FAWGi|mOWA}7-g4t0pJsZ0{~qBOypln zol2f1OJ%gKyWaotb$+dgL*)D#(8TE7*-#mMmEeum+=%cjC=pTr1Z? zxxuJM09JrY>wPVKF9RQ`z;Ynk?pzn;mcjLy09+F0d*PNvUOKkgZ9XL0SMdoF5h1U z7&AJYT;juF&Yaoio8fD4kGyoZ`H`pc?J(sJs(pn)(Vz?3|s$dEG|2 zctz`buUP_0mFxwG77 zR`0tr+*!%Fi{T7Z0xzoWDOMoI{brd+*_hU@DHF$Oblk@+?W204tvvdb&i&D?FW4|m zhRpw1j;Ulo=DAC7M>wMslM58PGW9J61Y zrhPN>J+r=j|7yvgo#ETl+@bn}`xRXEo*ql2rCaBkbG%-shZM?L!W`%PSMbjJ<=a=h zfez<>_qv@{x>Y4JQYjeJS;1fDv~PEOU)%M|wU9FRKB7~#@AkD@&S?3t`m$^OSR_w$ z^&QXW!F~ddoI5C&lx=)x+7Lf6{0GBc96YC!9Mm%XROwd60a)|i;e%uLsCu79mJyz3 zbW`3r=9)6EVK9Hy{0GB+Jg-jf+J0Nx%Lx|o?#_0v`LXUa{V_ z%MDl;U(R=&Ug^orN-UL_^-R)`Ud@j5V?4F*{zFY7B6i`Mlp7=1;@ zN#1AUTdSV$Cb=hD_`3zEk^P7MlvcGrmE#H=4Vu+ia%aBBQ*V5w6KGiSq-nH{W%e2U z{KfM=!)+ki#-EI9CXK7UZ^nF_QXH*nd?GFThU)=$4IS$R-g7)N0eVboC6&~P(n6hpS{aKvx#m27Zexo}NEPeZJ03iNq`WvdXom+b7ppd@-fy9ZUN(LNH zb2z%wc=k1``i?yRus0yS`^a{qb)HfsJ)HK}@ja3yUG^wor3EsbabsGKsouqOe4zW6 zDC@SI0{Kqssd<`G2aT}ZhhWgEKD9bK#GK!36X5taX?b5AkSO5+a2>nE-vDLz#^a@^ zYTx>@jS&EB^5n^6^1FX{qAb}mN4kuDS$^#Nr(WCtJmFpp?BCQO@S2H+KthDg`+k?t zCbsazb+6o;yR>m8$N+)csH^rn2Lr%(Kx1KL8H$!{2ZB-&=-6?>C+j3 znE{ygPZ-nojvP6xos2{V3`>XxGaVQln^-48NFo5T0j@cj7Ib(3CZDOUh-$Bi3jz(MkXDZ2n4l+m~tdabaN`MJ$GwGx07 zb}m1+G4*lUv}pn>j?OV1Tx)co=p6EkjnT>6v}v=E^srQ!HVa5bxwq3nZJPd=?RLL=$+JZ-5nagr znkG7lPGhPl*X-}QopIy3ZpxUJFUlPN2G183A$1BCKiBwQGrkdk{>a~uKjs^ud2}lP zxI8NcEs*d5)L;Mw-DQ9qw`Oik%sc9RbjF|$OjsPJ zQX8~>@ywlVp}X`umc}uI%A7oge@N*d$-(!?>D1QOVL5Z=GIfN(R_-|_WAFo`ZRE@N zreQH@v@|EuDu7F=b!?o zXpcViP{ny;s}Gmc{{Kw|d?vfQGqCIIG)X|fte zC>JhXP$pwpb7-Rq@0iYi-fV@ulG=A{m)S0mqliyN8M=cLk`7@CDol=Nl`!k+4`)AAoL)5L2EJtsYS<}aNscGa9eF8>-;cwOq2 zY32oZ$0i+r0Pdb+JEWaDSo`j^6(+g-3J$b={PQSD>+4dQ_pTHr++V<3D!4CDH~7g>+rTK zy%>Gbr4`*ibT&_&IjQf0rqM-{N`)Sm{yLB`W6c;3(-B8Zyz?Ht|5-D&dh1`m$tPo9 zlHSedn6v=R-O>%G&qod>*%O|Lm_l#2SP_mOS-jn~09)j-~&V3Sjud*`7o^69u1@?_bU<&{S|n&h8fzdjT76Ufy0E)DhBfBT4ZpYpnNsMFhL%!{f)jDts~9Xz?$x#@Quhl|$fB9TaE zn92fGy}yCDB_(hOnM48Ocb@Fy8C39_a4dGX>s>4kjW+7~J*$jnf7yE@XZR&al1R$m zS6gr6Mn{+^a`D32ylUMbMGOd%Aah*-!Hed1w zNDB4(p}p$wZkHOx8=6C$0Iiif@CVv+z)Mwf_4m;eZ_u3w+P8L(9 zh&o&0$JJvbMCD~3QYQY5wpf94#M&CrB=L3A^_fHvGo@fHY zWCS2gD5KMBz!QBo00VT)=q#dJ!l(otLl~M)CJN7!DSdP@=q%8&qO)j*1QY5)<1t;X zclYky1}p`*Fy#{fSARD{`E*W|j<4HxvVxodZfw){2jIpwo#^6Q5R6WvKjWGCW6=9M z2H66uJ?t+|LWWbe0H7#;44hDhm<9}sJ+Xo#H$&M=$oqGu4e)eco%8@WbWmqtJWs$| z@`v?X!C<#L2N*u++*w-d`x@39?70s={4ikVU3B-0<4MXo`Q$bRN(&V(Yfvt!3xSM5Z20XN!FbF_hV#Wf4TgbKin?YRipZn$8 zT%ke*^UMKn0R%B-HJ}GOS7*~OF!G=R_}BD3!?%usDgZ|67RSfdVE&?ikAy(2TD6S4 z4FIk+>N95FuYqX0Gf=?$?}g_Z>Wn94H>qP$f5tw}4;al%5Njt@@BMKFtz%>MTZl9e zP6(|rm`GyEmF7mO%)x z+opCe={K{zj9b!6r)<8bAZB=87RysXLEI5i{H}_gbNX8OdS)kCrYyE3NfOIfGu~6D zL~lu*QjMe4wyRmJo@)9Pi=uIR2a(yICgUh#+j^rb9Nwo+mR;c!@n-mTm6-%(sa@QdzX zuWcvD;k6?U8)%`PR0ieigC3Pfioc-D&*UbRl**9u2lxhhnlqPA$_WJ->Oa^v&_lf9 z!FK#br|JV*cGRiuEoy%1#aQ%;?Aq`L^4Vy8XVRt$ub&|~@+%nR4W8I|EU9w<3nLeP zsxnIH)hqlDk~C2=Q+N8jGFRFSuPd(&tRc@RQ?;BrX>vL>%?Zx{06+jqL_t&qvgwRf z$4K+mO|+W|%xsbzne%$K-7EU#md$&XOQoXqJiE}+tgiAIuh#fbn)a(G3)WASM~m0@ z?2qe8*?X(W`wa$r)xp2R{x)^MvM@=TcSJ9}cN|z@Ll^dtWgBM7i`Cj20GF`$>-LkE z`&E`Nr@te`^OTnpr;f>UpM|si*>5dz;_yLDw#P3UC^J_KGr;QDUj_)%+bb1*N?y?S z#{+(xiIG3wssp}EX(L(i+YW9tgG~TqO*N0o7I;LOd|gJKuk@}t?d88OOaD0^>$$#@ z=c~MJp82r3T~)s0N7a??tM84RI<38ceA#?@lvx1z?fbc;^!hU#vmmuBI zZ&oKsr!zo4suvD^zdyFIWJsA=8a&)uXOgrgs>hC;IUqf!zGn2$i-nYvu49`@vl<`k zyYXpH92^SJ02p%cGMXRBD-q>lEmUo_G*t1jB={)?~ zJ6JJ2)Y$Cz#ng<=@qxyra2sZhE&&f=mR;7D)Lvy=;>nm!lha^xGCdv!6l@=k^)Tl& zjho#ln*eA`FD$5?9_Kv-%jeDh^lts@a}Juow5I+b7jcg|#PTI!a_ir*?>k`72li@K z1(pC6?LgA4Tn7Ocd2i$B20om&7zq?CA9VXIcHGGHo$yfC^MaHc*3 z9>%2Y4%mE7*|1pW&YcYl9AFrbHY6m(guDNt3EN-ghLF9nATH;O4m){Z z+3N1Jc;7JF2)66xE0?8yz53?9$wQ=&Ftnx7c1KbH!?|pz&W=KFo;Hv*WE1GrS?2SM z89y4WUMp(>dl}=NG1|j1X|$iy^w0{nG7-!M1H`q1TanH6g_ysXEJl;ck=#oLkx)8e)!X}@{ie4Fn1YQpY>M-jZ>%;kjell_?1+3 zBFP`To?lYdF3jEK>Losp0w~rI$|aQ_=`y~l>r%o8LY#pa(`1oP>JN1W#kdah4h_$D zN7R$T+F5i0Rp&3AleIe+$sJikblPbt!Pym^Ug>cD{c?MY$Ae$YRby@U^=uRTot}r;s&+!F znw(p!Rhpk;eC~isK+jz{U5z_c5P+{I`dqE5BTCCh_l=h!lRuTWhW($s|J@T(OF{oy zmFjC<;^fvLISAmW_Y)LoV?$u;?oA4Cek0{nx-FR$X3;uVQjfSldCiA%{z*xvZj#@N zY}mHO^nc<8NchLHDON9k{^KjzwP%-HbjA&Gm+J2IN_AG$O@t5c{!z>R73rsvWncdM znv^S3QKjoHmo}$e8wA436|X8awGBGjq!QU)Lb`R$=H+H=D}TO%YBX(xmCHq{;=oa* zQVGhRsRpKO*`cR|q5u%xx2fF$q|d3-%IYwx8j7l(*xyv54ys!}f# zy)9>)ctHx|glsT$zj=_9Dq2Q{O!-vRj+8Ormv0@7`E!``9h-ErvERh^q>u9MzDr)R z_W$4hH)z@Ek?(}2W_%Cb#(?d#QSMgWs?Ym?UKj$KuEr88>ZT3t0st2EDIJ?VW)@$Cf`%I<|UJH_FP%kADu5r#`<<`?5Szzg9B^z||<3f^dMF)fCvO zQntFx`g^iWT{2e2YW+E+>Am-o7o}dcGaVc3USE&4kC^(2{4{&0N~~8?e+N}(^{)y> zFPGCVX_~@s=#O~h7YW(Hy1@{M_N(q^ZNjQrrDh~A((bIgRYh0t$eI1+Bi)pl zIDe=-cG zdV(%_`XbuvYl9zWH z=9d?48fa>Ne*E(rRYCc$e4;9lyo642)FOGQl^UW1#P!x`q;}~OLy1!iKWn=$swy!G zs=>3_%cscPrPJiOoBE{iiUjbNkNjpEq$yq;p&!XJ!ZadeFhLR*eSIu6ypzB*KJ{N` z0SAz@;Nl1%6Mq(KEMfrS9dY!K1r|qtk^FJsEsGbv;kR34@Qu|0tp|?y4SDb0XZJ3W zn{yBK$8U)D?n-hA??#?j_@kS)tV$9Yh#knk`_8}n9LP)#05rL_f;1aI9Q~y_W@fb6 zDs1u#2;+<>!VmaDd(6q?q*tA4f@mb~GLm1y2i(GtnfGWo>Jk9(q9u%>(Z|#8BVmkG zZgwW>3*!!_M%)p0^nAMw0n+Zq1NwXS%$+g7-M#+2?(Ti)clZzW=ln$L9?9RKLrLiZ z-0;Tl{F4tZ)FS{K0_GjmMbW_n2TiR3V=zE2ejrprJ*@`3ke)=cnzm(QeMlQ)qt*fT zfU^LA=vajSaA&9v{N^oM*r}U(_38<#o&YzIen#yL(yG0C_cjS#K-|=HyZ`jlPiA}* zy}76YLCTzC|HFn2lV;7D*=EAkYTz?ey$lk*zfPWPy5<3l0a5{dhYkBSs0Vo3v}uCA zUZ_Wp9>ymC&2WUYDgf)N-Cwl}z#ElO5NT|$?V+n~!Z>1Xv{O$Gv~7E_0=;hq^}O>Q z;1p?aP8F0dU%^0J+&R4zPPcqoI@U2@^BC%MJ;_~(5b5nRhC+?Lm zo|3{Xy+7Vn;BDOO;R@EwN+CssdbTT@+%K&Z%wa)UqHrmB@Ul1L#N$qtw*RYceMIS! z6;;Zxu|-|8qN?vvi7+IOr!V|o7o}f>)f-%{_^~Se_h?X(^!se|LYoSLOO%r)#_26*1cfw9nhHdYN4HesZ0U=viMJ)PAy1nf&to zf=}(j??LT*5)0|3)!xNnKP^xque>(8ljQj+Z`k+F@zB8oD!zG)+}f*=bBcH;9seqL z0Oie}SFU@%zMhi!SMnDuAe}xx-`oqAD`;*@x6P8jw#+m)@4s<>4X}E3NP#Dir@Sf= z?)b1@`O@o`XXUdgNqyEk;~oiq3eS|yMtKyJ;~U0J7fzl6$4KEK3al<4BER%X>H~ux zdmc#Np@WBH`=Nhi;=g9Z-}35^PI5uR8&aqN2nL*ihn;(NsLtE_EQEpu^GRzpa^|Gk zLxY{G`haU!Yam^({y6ki&`@82e1)Z?s!ItC<5&e&iw2HSA7A+txu#iXxkLeS=)-%) z$`7*!%fdCk%j5+kRDb*)m4rTEpcUa+Qp+^Pq47|@7*W)3gnY( zKd2`M_Z<+*F|V%2U;XhO)zx5L?B6lPbZN0);T1QBZc~ z*(8Jhd@J~2k#dFXB+s06<7M{Ralty2Pu0@Uy#?z}VYOAD8&`d%J#SYq0DDkF&RRd- z{!yjXU6-hJm$H9u#EmJvM);MH{L()27S~Khpu)=Ew1~cA^rGG=ynQ{ZM~dJ4U`(VO zf~5J}us>yA#mT6H`_-^z(WFzElm}N@l)qE~*|vYZJUys`6i}V6{qIQM?P1{;E>l3J{5?XZE>r1$U4JyF*uvVvYqt#z zZYH%jCZfO45AS1ztmq4?;CmWxv=E&+q;l469Ow>T-RHN963|n+4t6Lssd-CH`Y{Vq(n@Lns^p zM&}+!g3+u z)2T_`b4nXT+7D0%kd))=jsRFN9@SUfMvNS3dT4ov4(5IesxDl#$UsKiFc1VV6{??^ zcaG?XF%F27X{2M+&X5`E7G8h-bu*^OQTFrBJJ0ml(x)R8h_S(r9Xndk0JPM*KU5bk z^$su#sc=9c`gnjEE;bmlbDzjOW1ps-aSFp;7^(D=OHxOdzk`VTi|A5Z}IfJ&26 zC)r#Lm83=TlMRL5y+2dKdn1iMw6ly7;I7w*Cha-2{|cwuJ)`mRJ<`C$?TtUZ;UZPa zCK_!qRzjk7@#4iYO7#;jUAolTb}VUfI@-gA_eO3$cdUy??t(ot3oD$Qx3_ z(T9_=JP72T^PU90j3pVV`bE7WK!rpIFjWCm6 zhHDq7WT7rW`@$HBi%dLA8)_o(n0@&(a6{-QTvdc+Qy4pd*PG^>S1&y8+%ZJFT&NUbO&Z1 zCb#V!ditPPp^|C#3-E20wkV^_QQ$~2i)Qi2^S_eNOQP~ zfHV(81wjP?pa4fWS^TqLc1Jsr0&?NRCsnD3LPH@w?vRVyxx263Z|+>=gFLdx06c@- z0^EJ$-+c~bCI_giTr74sDDYQKrQTfqboDqhC+uJ0#CJr47!UInyK! z;UNPh60GQMMhX<2nt-FT6dXl&sH12k@0?%mNV;K_-t|v_Am~O%+AoxlvIMln3Oxpp zApB#HjQ+fP_oQQe0|ySU%@H6x z60X$8Wy_YywyoRjJ8v4C-}Kx=!OGCdB4K8RZg;}<8vNvUMhJ;xCv_ZaIIawt1JMp+ zrPOq^Q-`@?)%#8wQUh>#pPoFsdc$S`227#&wGay~*ZeD{;8*QqlK`;LOnZ%#In@iH zelqKKtz!>A`s?BXEXZsCm$>e1@|cU#zxy0WHwTbl4X2tToQ~*lkpPTk7XX*^`RyWH zDdldrLb{QYQOa%giW4_!ow`XcPw6w*e9g`jHHb?lnBZ2eS~aLc%}oNcDuXP(Ic|p( zL1r2X%c}fkG+y>{IEtB&kYHtpAry{DqE~L*xN)W*jyRCw09*j1U~$hv%`Gfl8YpYG zz;WfyZ}*Hp_dC>|3nQAl`__f$zVW(y-?8|0zma}n;ZJ_tqQrgg-+c~bCI@H-TpW|F zqJ3nUz|rN*j59lh?b;^>wcr8d?HjUFT0_%Y#a<2Fd&7|QPKb)k4l76 zrII9Okf>g*V>@8CPx0dbZG?@ITtiy7XlwK_K7gBRO5M+>&jFtUJ_mdb_#E&#ki$6; z6Nt-V0)2hoe*3Lu+q(!k(tNMj za>tBg4$%L==Ri6*;6RwsqemYm{g{s6a_lDwx?#i-Tlx_s|BrS`P4X_Ic}J2tJyLtb zixH*s&u?aB;nl0lyRW|bid(iM15I^Im^EvbZO-x=J|Kk~jbyidAIO=TpLpFen(6d} z6&UTpM$L%Gn9Rl_YCIT|y~KuUvu4dzGQ5(NC7S_A$THTU6Bj-30J}bAj04n_0Rsja zKPh8u2wZ>t^>#8TVg8}v4%w(Kt~~}Fpd-<9f)*yjT$u!NFZJSi1Z#`jg;Q0#VnHcmyB=jG^ASD<`BsqiJ;QjCRVO;p&^FTzNycZ6;G^cq={4pz#Q4p|D}H>+7zZfVYp=b=u8f70bH1iMu-R6= zyc+URNnFSWa&u1k`6=}|;B&y|fX@M+13m|GF9)IqabXdQD2_K`a0IKpx7~J|n$t{H z#D1jMhy~+PrIPheA(87-N)905vsrcB0(c{#?2N-8$%CTgLJH*WQT?4Pl99Yc9R+&$ z0B<3z%tLu~!I)Qp+;}fuE$}$~!x#=kUM{?_l^%0`$E;;z5j(He*?7{>p+ilVC5vy2 zpu~d!)Hy=>JBLU)K;T-9A2%Vuwa8p-G zYCD`hZMs<^4^3Zsf1{Ta@D@-br+~Qs>Vmjuub$GRNfWayPG8C9M{Xjw*@IC`EYwqW z+1-%V(ONzjG@CGCg6UO#{`uzxH5Y(>UDdE!Q8nO}U5R#tF|T>^=E-~Sy=UNULG9P` z0pg}}9Lla^7a$s&ri{k`xe%KmoGfDFmCaVhG{9TNZN_kZL#UWI`|Pu=AEv#<1E98m z{eQWJ1JoHd@G(|qZA8bc?AlgxO`-T1^*P{kz~_L^0iOdt2Ye1h!GWkz27|%^lRWgM z_E!CS09i}~k2E1YfW=jxB6A>7%jWB^zm}FQTbh(b0bK;MC}cs%QOr`MOWQF_j$$J9 z!;#x-uf5ic>6~@eS#sB1ci93{3pm?P>;ajb(GV8FUAuM_tiZ1i0MU`T6x|!7nH>RO z+UA9Gh{K%)`mML#DtF&~w}m!P%MGxYqqWtmSGO`mx-*n8P&7_;8)r5xTC~X0K>FCc z0i5Ic)mLA&r~#7WN$3NGA32F;EAZ5-XD@l|vB#{v$FlLGVT>3A@g2x|!JX-^}aEbW@X=a10n1apLuj|-FzwSU>SJ$GwA9?Oa z7bEE>zvT1IJMT2$mVT8@RorZx*72D!8lz*lIUpCwUlIYfAuW+dU8*hw`yiI-hDq=z=8GKSIdB3Kgxobk6e^$9@Y98+FiVfum?@~M7HkP z9Jin|E4^#CuC(?Tw}c+yfOhQNDnHL2Cg1)3mF(IVoqqmk@|+oEc!XJ#p|N!7`h_X@ zc_{IatlqNxNM|AL@kI^dvVd|I1~!-gay0WuF47k-UTh%Aq)C%ZodSRfK*6wK!^}|2 z{rBH*3_{NrR=V4^Z5y;YjtU4Q2{B^BqUWK99x~~q8*aEEXaPDJKj=y$T%;e-f9VLR zN%Y8m_St97DNFAd=lJNO4-E{X+*kzXS1BGOc~}Sn+#*%Y0+EYn0A3_Nuejn0sadmT zj#?PHy2%|$rVbrC7}$uEQ$~n-{>m$_qzD4gH8UvGIylE1AAR&ui|YON-#;u$$}qF6 zg}cZ&mVL=3ml!|8NxVPv^fQ8rm3R|*%PF*Z^JW8`=wHw+=l~}>{+&}Y5r=dFS{rz+ z;1qo`ec>!s)8doQfoM2DU)xtVt56YhYEm*wI=JxhY??HKg7D{`+q^@4Rzu^X+JEW3iMi zfY00CJtN1Kt|BWo{UtqrdRbckx0+pVj(kd%|MOQY!O5V>AInFhpNZ+4qrr1d%=emZdQ<}x__V@qMb<)35rEE=k^wak8(T^{t`Av55uwcy}^8Da?tzNfz zuDaa(P9y2k_eQz5Iq5CG5;f@0dBm<$;eclCLJd9+z|;=|BJ# zKl!=4)T@4`jGsGPu6wP4Y~Qmb)nr7Q=XjT4s)@>^m+2W(`c8aDeqHpF%v(KE?s)Hf zd1cr`x+S41IFHFI%Qr5OAO0AUNvSi-OmOXkD{74at z7Nq~swFpqhGgiR?PdTEw^{(t4ibc@7XaJ_TtiKQE&3=sT?jK#rwri{pe=c3 z!RRieeQ@W!EuPhHP9Rq=$t!t$_~C~w{TTRRk(^#aDh$1YULvl{pm2*Q7WaU;(^Q4X zK|OkVq=}>0NaAAAJ`y1{w@d4OJSnqy9M3co7OF6Azx{S;)TogG-IO795H%#>$}`^a zXQ%KS;70xdvrz*=-9V4BPd*3I;sD0XI6?9H>#rMNNqwRn<&?f4t^9?1Pana2ip^EP zTQ*bOSd1htZUuGIHD6#{!$0#dc*FB8x7;E@WT&nlm2MPPj<}% zbE4jQnvTs9=6h_w<))J(T)pFeon6!}y5dHSZ8x@U+g9Vow$rBJ3ftI5lg3UO+jg>I zn=5-g@A;j5&fb5){b9~~%rUNU0XPck>3#*B169Y2t07?HbBsb`edB+~f02*6aHmc| zgV6kgbh$~5&eg5{SSf5Sl`sU)KfQU#ZdJwPr8{_yP`KYeI7pCY7{$KOOpi^baj+hd z*H%rYr86Q&GXrlcG}&6ZG~c( z9!_Eg!hdJic2g9Ke8U}Pvj?ldf7X_~ho9+8DRC6E(hh5m>1`J6n7@OC12!&6ybC7W zg+|#bpRrDVad^BO*4unIo#z@Vw9nNf57qjZ4o(tNVjafSS4}%RuXWPP!nCsItE)^= z#Z)X-vnJWwFQbriQ<3X;Fi7)_2=}nch9KP&&GXXFBar&^P(#v99T_0*<($#5Z_;}R z!AC68xo3Ie;m>wlKEvp(k6o|pQ%s#xzlGPRfw=)U<>a=f)cYNXg~Y&%-adit>SVAv zg}xSE0uz&a5*NiYJ0MQF#^kc&HXbSw(5m8G!~0<;9DFlESCZ};PI&GAa$M?IclK^w zy9n{@0p_$lA7rAz=)c|_QyI5;03XiR9Q^H=|27&~(8T_5TyIf#S{x8S_LT*Y0v0yo zU^glFU$?{24E$Eqd>}P#oz9C+sAw8^C0$BM?rDGSeRrg;p(hTz>PIc{_^0_j-)#pH zO`cp+u?QV~)El0>7L80rC_2gN-6y6Icjw34BD>2@#Fw&CUYcW1#EWO-5^CfzBfUw^ z`PQL$uea&vAnU+;lpO{5TJ_o9c-#@TY=4z(O73oMNCRyT_2!Ugll1)A${j~6x(-Lu z_zu)$cXoUmFGD(Qf^QQOxuRrcWX+k{v`@1Z8!76S?`I7fO)dH0YYbX{4fF~;poYOL zXw)JT%Dlt(zoiEghYUfOo5>UcHqw!4joc@ia>6me<13>5&9Wj?s95vpF5r~X>Yq5w z%+g47YdbFd&lX0jn5GeMw!pP7r3dY1U47~}3+p&@#d7(W;@tov62dKvw`?rZ8`8qe zB45pT)}nUnfwr4#VawvL*KV_tG(!A8vy#gbora0ewb+6^?s+-`%p@79xV$Dm>EBPy znxX$r-Rz?X^k(b(26nmHf~gOfkf1kU8?lCaKm4m3NM0P1CddTJ-&1MuowOw&O-rOY zLqGjh|Mi^scQ=E4OL7s95}zS^ZNSY~7_c|Rbmhs5K;WKg#otABnwd_xKY|NZoFn^+ z9ph}VI+J`t!0T+873&9FY6fJZ= z9}vO#&8T*Gp6cL9WjsR0$BFrfyg_m;%%&6tRZGAyK|ckXm^5zpgsYNnUj-g>553t!mCTAySd6xaoBlDA3dT;32 z)4S*l81~<9QVbcZ4zdUnfGoRZM-X=P*Lg_S>O+-E${NP2`-N38jmfG5%f35#3d~qTcKyskou#axw&a?*QhnF@TysQ_H`g0& zEub;BIkw&C&KS_X8MKh8+m)X^bNB~w%$k&rmMtkw6A`8=*86Ve2$qz~Hu3m)s{DCm z@>u2`z7OwPhS}A0obkFlXZ7k^^Bb3YNJCYyFFp_PXOBRRlJ+Vvors_NKdB=@D-47J zM}#pRmT3_IyPRZc`8dUf?@MPzX2|rv0w1L}JdEU1R0lh-(_88p5`}yzK^&DD*dIO# z{IstRTbbUDDUV(LLeeFW4)8aQ)6agJz62a3m(%I0p*PJ-&AvE6e(tH?J%iZdIsO(z z{4OrR)5=qI#{qP?*z{^bMha51URkhdAKAO<`k})>u#o z{ImzhcrVu^ezZVy$@GgPm0nY(ON|YWq$EEg-uBa;{miWAl5uIs`K$P4|M#y~AHp|3 z{+sGG3{X!XR|!sx09r~j7f9@djhdiNceCNWMewAUEtcRZ^hxZuGL)DR0GoQ-*0ka8 zIzXs+P!eX_Hh2_^j$jxc>M6*ohn)7rHsFRIfN;d=+;&XsP}ppjS+Lc_)BmeD(AS9^ z!C)v!_l0QC;i4?LAos&)h{y+jLXS+*NVe}%OB+wCLaV;WlJEUgmXbu)pSYnGQ2;^w z3Oe(1FKVdK?4kza7KNs0@?Pa48sjte%Leb9-`FFk-xwIie?MFOBUPceVoW>E3HhED zdFhRwD(F=sttONCa^pGh>fJ2vbUGiV-fa$3zuoHhdH}dkqx)qi6Lcg8Hyo za!%X&rxyoz@N$e+aSPE|;Z(CWIqg14>~M$kWps<^VC-iw?rER4YP9iN!Mw=BS5HG~ zD(i<-0>z+|bZ266R+N(>jV0e={must^NVI3gc|#Rge=X{)JS1N8YTaPbC$zwk7%Qy zgNa_Q+mNm7J-ns#E`?j2 zk$+kl5SFt)T8vc`Ffz(<@))F1;sfWqTh9wp+Z89^C~mHx!KYU~X#iGcuG1*;GR?nC z>so{F{|3w-+xQ-~+gSbVs&!x2zv*@mWfzW{dD{7P)xs}n$sowORS@V@K|bsNcQZOJ zZE5ugCK11aAxVx|8lNc~6|A0Yw4bKXLzDbx((ITT@V0kb?>pp|dbMyP@U`OU#NdG(V%J-PWaak{IaQ0Wp;y)neqA3SxYMR*L7=CE3Ml_bhYPPi&zF|w@a1z z@(y|L@N;qjLw=xc<%b+FOnQIp+PVBkNsKV`x|F&_6vp_r7qc{xx|ObylevUnOTn4H8q`J72Ush4~76h}6vr%+z8CG@5p{KaOiubqSrV@sIT}Dq$w3 zq0<$WZL9Eu1_;X5J)}EtfvW>E44&KnNM71OF21QqVrykGwM_S~Wl04bdOBV1F3pOx zav)_{!(g`De-HC)I)%J&fSkJ0Z6Ni>5udAWWvSI(Q=|FN?EHyr z;VN^)7zP~@pLn48zsl@n2=?mP_4YWH+2DLa>`9hH8N&1iG%74qf<;ESn});1PIi`|xXIObIV7^wh;ONT;Os?{~x~Y}M#@ z8khuJOBRqOK}&w4&P5rCJXYpE1|A?0Db{=nKBqKycouAE2K8T~ z&D72zs8&j{s>ljUh*FV+;Yl61;m1P+HKs;m8QKr(PUPhsnyP7D9kvxkxizzKG-$SR ze~S&X3^^iug05E)U)P;?To=NhI;gO1;p{is#R{|DELDSXnZsT0ugqDtZ!MQ)b14Ll z%gEa|B7~$c)_`42yj}5cNqIS8T^FJi?Y9_kC7l44>xeXYq1_2Pb2b_BI22ZtZ?jEd z`%C%^GjRq$0#57w2BzoZ3E0ml#7~Xo>)ZW+Qc~MXr||;ta)||PzdskO-8lmrT4A>9 z4YZw6{K%nHhG=lt1K#y*HT22*7R?6gy6dIH<&mI}JX9Cl zSWq&pXYye@9hr9qHmecCr>y(zr<%~HY}OzXlXl;b6c!WpoToRWl~(6WY!*GeMS8Uk z$07J+Df^_&Tn}C*!b$?LZgYbUo7W@~!CDs#< zq|M3PhSWKGo>T%CeEn3PhMkh`Q<@LpGUMcJB7(W2e%r$ zAtGMz`5x(1yU*XVM@$b=MywhM$JQ!B4if2eM;kLPSabuKIPv1^wgT|ua>u4RXb$R; zCuAGCD^qH#GjF2n?dLdNVbL9y8>Xu4$&(4T)W z8L-cJz_;)19A!<`Zrf#Q*75?<+wS`DmPYiE+SXk*-*4`4E>~X8YaTOyYdI9A>TsCG;CFNB9sQR%cN-s&?z2Zd-GjgYw0?B-+?- zn~x`1OpUb9xtBU>v*;HX=eYstE^J0{8_Rb=?fxIVwV&ez^>gfXgl0_s;~tdf-=90f zeS>nLh)+vrGLos$>K;l{f|sIWwDmoj7Kc!bcHA@?Wo}-kU&j-%wu%NU-y&x7gV6}T zbOuc9V%Khxu1MF|dQ@Kj`P=k;Vr{9^=Q1dT*+AT+jlDe1wCu&+?lq;jaDFy3M6T+7A2e8fpTA5Z7S&+oj&r3paxKTXle(LQ)eeq-)8e?HZ)2g>RL;M zs{6u^qUK`BIfQ+PYvo1ps8z+OfrvV8RlG5%m^Z7+C!V062G|>^1CWw9q`*Dn=Rs9V zU4bo&B;{kxpTZUXMLZFahyt&5u>6-QKKVQ8G3ZkP%+?iN92p+zY)t>iE+=F0*pFPx zG0hSso^s82%lctUSMLU*C|9eU&h`!%SUz&R1$g|>-b51FS0GZ{VHn<@Uv434t}bGG zdu5FGv;H{Ro;Wk;TS4~=y7lU(m)YK%1lL%*o(+Mo57-(_lN%d$yqUa?DnQcND8ED( ztwL*MK_s;4c3yzm_uIqu>1O2ogI>i{M&0^cK*h|H>sC_pe*_8N`^%41_a^@ap%Rw< z@NIF24FEo`Q`S8*@pJK|i{}JX%jV4w*$#egl)+C!SUyr~Vg+rA>2%5v_iua-hj8Z7 z8>Z%>%%BQLQg4Mlh(>d$9})2+tddj5xdHr?B7m@7{$&-9$SrHR#gO$ffc74}z<(?n zCB*Bllq0w8AfvLlgD|Ps`SXTZthOxwE!}~7GOaR)Cz6mm=773=+Z@%g@KAO#ujk1; zy9vNRr%bJvI&G=i(1s|k%)2*ESw+R4&qAeVkp3O(R{w5@(>r7v=9Y;^dWZc|O}2*~ z<8UZ49EoN${LSoZ`t?6W;M4v+)}_tMHUv0dM}a9+{OxB6sA+0k7jZgYZ8oOycu;)a z9hv=hy)9}cEfGmgJ{ZaC402|W=GXg!`+36$98TM5C))RRng=j9SuhkhdF<_N)|jQ-M*hkZcpIK_HIqxzj6&B^NzuSDOK<15MxsgaXiMbz2{EwVdp}_z{ zn*|l1 zhd8@CUkQ5s9jF1x1=Lka<8 zmt+#mU}8qr2EG2^5`3P;mT4;h`EvPoA7s6Lop`Zg$ZqRTf|K(Mb83qPzsHzytSGfA z^ezSX_Bk6ttB?g72|r&?iT65Zl=m{el$qirdNg?aTZjA&ExE?+sa!hY%5pTOYL^Ph zAsc^~?HqAY6O5NHRJwP?Y!2bc(rdt)iR6*Pd!C0h*-D;41)Zg7n|F7J{bNAiRo&;^ zq0wkD}ja(j0%0gctTkA)5vi_3GI_ej44u25}`ihidXJ z#{|vG@T%turNegIlXZSQ$BFN+Xm+B+f!Q#BI@Nfkhu-x@7vL=*eOr&vu-ga-$E{YWpvu4edwc0YL97H;npL0+R)YRb>{f+O{+Oh=3Wpjw zAh-vpI^rX#6&o!w4Px&}vOGjbj>Xa*O{Ch#pWDIb-NwcwmE(4c4Lq85-Y4UJySpAD zTib;(`%!2`%!+UqH<-Yyiin8V^vNtKM?0dQE-8~(_H6DDnZA4$M`;+!1+{;nel$*s zgqsZj!Igtxu;;4{b)(S8YT(aL428udcy)|h?XLfpgyo_mi?ikza^uoVowzt_dy2gW zzMeOCibn>DfKoKS)%^gHv-1D5pv&{d(t~Tz+&Wy(Yn3w>7s1n-D2ZYpXPOr~J*uMbU*9i+Qpat^i$$%$ zJwknrG&z9^nJ~21lYF&r1)Y$}BLA~eT_a@B2g|)^AOd_`Pv}(`8$1 z&gx_5`v~S&KyBHZxb|%OWf$ZA@fd4&!`&Z5L^OO;|N9dwB8=q-Gs|CF@qqmDwQgf_GWr6)g{f({OL;Hj z(xE1Q(HjS~KHTi+=UW^hr7cR?<;+~a;sEv@vPX9loza8<|g?+Wp0V%95hpkl# zcM|Q7=ko1bm#WBZWi#%9@1yhx_M1<=EiQPU1ndh4RW4&I7k0@ z)KOn0yCcW*Q9z#~#=^}Ud?;wkHuyiXJj-i)i_Je;(-~V&U6>yI!c=_6=d~&Fu$#hn z<-?&@8%GghF-TVwGtxu%`--au6Rz;M;oYqd{1EBGIER>;bt20y&OCdm(QLOQs(F~H zAR2d!XY!ij9#a#s;r=P>R;l5an1m&4MC-8W9hS;2{0dotSD&ecAQ~4Rjl~YC+S=E( zZ9*2TYKK%eMB6Bd4r6H(C0GeUb-yHEu&*NI{hZ*_j^?+++qb{Npr}QLc4-yK@E^pf z^s3nrA_3t@h~QFgAz3(8MlfxD`O# zy+LYYqkhnjwwN5tdG354mpe&2>bEW}|e zchv205j^_wkGDfn`dCF$$AGI_D)<1;v*j5;BZMR~*~8q!vqa|qdbBQNJpd7g&vl$s zt38J0cwAglhIsLOI^MIboXF&D?gBpJSwim>&Z(PdpgPry{{t<+B)rgjq`3Qq@)J9f zHL8|l*gMp8+*IV@@o;Ac-7_0D$rZ5?Sh59;*x~Xl6EFgz)rD zr{G88-=!f+_VfAbbSn*3;UtIe?$2DQo8NHnK0hIZz{|x*?8Was|U zSR&ZKmbT%Eip^ZT)nwEgbw1-a_D$qx+!B^+nmRBpjph&q_%caeFO7UDgHeMDX7E$Pn1ss!e6nk#AhRsTGt zM`vC0#EuE!w7dy^QUf?PG4V^b@ss?qvT80m3*{hRDJf0oyA>1YQdx4)_X3sq#l|^D zGYI69)y07#YAW}c^O>H0(e^e$A#qOIYC4bpa^-t6ZcwcTXhM_LIM{b6MejuRx{pQ) zZp0i49>PsLF?J&?Ua$GVey;?(u^WRZBE|y1j4A?OevsB7c0ZOG#P*hs31T_uj$qo) z%jx3J1Vu#bL)a-;)sHPNk{l&gW5{Q=Ti?losF{9@@NOsRl35v1O@xLfke{4exa|rb@eNrkoh_4@ zsRIhI(k~x31Q?PdhcSR5%&XfSU! zmwBbnJ0EsdJZ5xVO+Q9kY{j1nDI4#G$dBkDYgqXgXw+Mr^RzfOR^vd=S?IUgZt&0c z_#0c#(&R%Izo`wrOJ&v*EPT37t*(G)Y$Cakt&pj#f}P(GAF3|u5s|L3(x}+7N|Oo` zVG4ZsJ@%I^s45b3{CUSeGKFIsfq1oox0qr!gL@GXu$6T1yY?ah^y}0t>5N`gBm-Lo znDxJv=RPvbUcl3+E0woW3;Ftb_g!5%_6^*Oi>YmhrDHwvmm^buzYoLRC)1=!DfM{Zs#1}@F zfOWwr*eDx5cKhQgeaIIZ48ndNySfz;2YnxM^b4qjMTtD59%#2}txC-Lt>)rL*#nb? z1%$~lsYqq}dnOC#L<;yNV<@9%_5SYtSgQy>h1}%P6h-dQUo4-g+uMWNJ*g;y!5tdS z3NVM*5Ajax6Ki1Lql|EA8*638D)M1n(}|tPpN~X^&^}@L|JMuPb<$JCYpd8VJG{{UXV`E0C$8X$SR7T1T_UIiaCHO6KCwAUeu{ zvEKp74a64TzN5F%e*B!1pyB`8A@UZ!ggAbxdz` zD>L5bdY#BxC}++zs&0T(5nQW-L&dlRE*85ewZE$m%D@=lu;!|nW3cc!iWl|xqK>L% z&0>wu4n^Ptq(EN^jru%nWih-<%=Yc+3Im?4@!N_<;wK@4PmOSLhZ{9fdWriT535wj zy9IN2$0~r{3nX+mE%ci-xEKLkUJ)8sz@{Dvs+{r-N^n_a2;xzTq&?9e>=0bG6gc8- z%oF}aMxP(9tkgA7RDef46b*Plhe`p_M@Txl#$HHV2tL!gUP_KjAEG=W(lZ($!2|OI z_ft9-fK*VT!1m?t_#xgh^DxUj9{Pzn96WQcNJ;_F23X%cI%gYBY=bnP=Ut$Kv~?=a*Phy1j;16eF*LUTrRjZ?#oNFneZ&fHPre%U&h11x~IhN?g9s1-+C-sl9KuIgVvjOE^4+c-jHNxZs^yWk6mGUKw9XT`3=%HearHJ z=0dmhzhQ=SMT~hajdIVnef!WCa)jw=c=&nR91nH=c655YLnmCOBpjKo*GBabD;sj? zl&}!khyq6)7p`8)4o-C|Ym`y4@^2_=5BWse%!qPlWn`4VTAMEyc7pZ7=HSPQ+MCWi zC_I$4vyJXGnw>wd6B9{2+(iqYSh=D9gs)>b=3v3de0v~M!VUQ}!8@3dXY;y6E%ye= z-!j}Jeoo=~bhO;LdnV}N4mS1O-dpq%^54l2Enb?cEz4QIq8oUy4eD_?tcW+aUA&>M zcYnmyR{SHG=A>6vK{Bgy2(sC*ej(Z-d~S&B?S8gDLRCPX-m1pz$ZV?R>Ao4XJAb%aa_G}``!F4>FqCLYqiqSOMqq!owfqYy{8Qg^H>Qg zOLLoFE8*hp?ERtVqR8JDf*DZVixuCO6<^UV{^lY;dk68Etu>C>`kbN8 z3AbPFP(_R;d3T@gnv9_;3y!h-a(S8$8VP}>{t>e#oCKPD8(gp@n;zWp6(aC> zJnS$|mD}$YAOu&&octHz7t3w-C0yRb084Lk&ZQHqSCn5{1wk_pCS6^Mj($n)<|HF_ zS}Vmu`I{M2EZqAReJbxEnny=Kj>%yl(3ef%?{n}WGeYO&PnZUhmU~z)`IMDjgxMGY z!mkovrgk3{v{hYepc_Lz6Nef#*aB13gQ%2~Nul%Lj)aq>NTG)G$|9+5;umwN*SzdH^voKI+K5h`6&78<7A}^@&I% z#3MXm*Kj0?*bIJ&35@&Kn=#gAw{2$r$3cGUz9Csb0q*=wTH25LS0Yx|_yhvak}$u1;xV#% zGAyMNFhmPprSn>?k-|ZFe^X<-MdQ@a?ob=q_TC|lXEfB5nF1@2etWFPrcb3_fK=UK~$ESnZ-N=E}xLBH{qSY^-lFw12)x+hfcQSgzC9HE9WIf>!a4 zn)t3b<)j^bb^L4T>ve_b*P*UzjmEQUTAY=SbnmYVgWX@J@`dfQJ95U=6M49{jwY(} zbb5x4)F0G2yEHbw!xY^g^TMJYm>HF{5!U@?J0v0~ea0^qkUlLFFH@XqnUT)jHPPw) zb|ZAQl*19!+pJM*V7cOH)~sZFl})H9s2nGOs=}G=$;g`sT>)*jTuU}4$1TL|wE1bF z&QPUyjKEDyvAvCOzo4oUwkO)-YN{Je4Lvyd!I!3yB?YsH8B5FgOw}EJWQoC%-*v~^ z?Yf+IG`4$)fxJU}0V5@ix%~qu-rZGcRcDNJe7$K8av1r=zcq@+*_|>`A=sFt0B=dm z!YqmiazIGNEJ4-A@Nz7zUl#-n8?UrFs2^c47u2R@_z0R& z4pAt^+Sk{HagiwY>uvGjJM52v5O}R-z2AtCHi?sVx&tXd+Mof{!;RcN(;_l$g{2f4 zqJy%X@)YpQyOF79#{Lt9DlCx1?)=_R+@3`4xY?e)Tu!*Yj@pb{QF#{Su9 zwN*SyDM)m|E%dN%zq72VJlHd~t*lo&%_H8vcD+q0*tGJT)-J%+_ooMIVc;NZOn36D zfw$)Idl5L_Z3Xt(a)7$^`EB17`(LbSna2gRqxagQ)O4_U|Nrp&eJ+FjO`Y*`6)kS4)*lg-DZ zIj}l_x3Rfuqf22@JN$=fd@$X_oz4uHGEHQsxw!ktE)4DK+C#O3zAmG#;@~6+i|FJT zqoG$#G6P6Ot$A+(-@@v>lpk{@$d$V~icB zRc|MWfZ?Aog4Z$MQ_i?PkM!%i^61bWYQtqIJVJU|sNl(f>q(RQ<>zBj`JpgWMGvOs zYD@7zZ4wRP5&_*W!LIZ=MOB0o-;ya`cLrx|hhk^`!pncu3GZhmv*HfMsb!SkTZ2#R zZd}@RB}}c3oeyrd(X%uLLRfA`|M6^jhFN=TZ}&>2Qp(NCn--ALt7>JZM8DMeXCD|- zDQZP?BTH7R(pB8W88e)xmnB;w>*DQ4`}MN|TLOOOz}TZOs-r^FR8ZR%gsDWEfKMB=HVoiKsbSCvZbmg`YRZtl8}%2Dk1^~aSq_kb7|)qf z4h0wDIA$lh`ypP>G!-d+yUTT!oUGzO!vwnkcQ|e4l(IZ+O3Sy8ws%(b(DolzF~Os` z{Nev8*50u?AfY;UcTyIGGI5wVTB=&wG{^Bu_rtUd=$U0=CTy{lwHIdZW$tPrc)*(e zA<|$b_0UNvxF9U;@74aEa(B07$i$kX=WiJa@irB0r>s60u-#92n|QjJj+PayD7%i~j51yfhN zlu9j-eRC%|feUo1+QWb6zNH?NTW!wyDG#x8)r>mPgs)F4&-DWD&q}4@#t6g*V_D{6 z5%Z<(t17Q`bbWO_R(Eqp9)WMbj?s4S=!r6&aC7=5RDj{g4I8;nK^?lf25 z-GuS)gDs6xU1h4d;i?|QiDNS{R7)m?eJ2 z>6*Mdw7n(G_0^M0eT==$0JnI>b5+`DnQZ_ z{;xx7dXt<3s^_#{&2nC}ICnnYrQ|=p>g5)Wi2BAfU3y>3YKm$=G-)qYfxC(2xvksv z6WN}aPo#R0tE3#nlw7Vu+kpAl_z{@4xyczD6yf~K*z5;e?^8q=%B*mN&&!j-?T@et zDW8tm?Qz;XPpyYqfnUXT}leD;RRh`7eeaia*lo#5U2wXZHA5k_d#Iy(<1r2$OzZ~4XP_A^wHdT5&9 ztf<0MoP{piLqm;l-teM)bs=eVa@|d6&2X@_ye9?TF>goiXi$vudvXE<01t_HMWjFl70p{K(UG_S>CBb7VWGBI!pfoqddeonLuv%S^2Y1f2C4XS{NBUx4t4R$EjF zuf3;)v%iAh;AjbYiJxCDW$jik6Z;q-(r|{^<~#i|zA#htx0jR3+AumzjlJZ+#L(a} zhn^>**(o^;Dkx<^$RwE2ua$1wIv_F5JEN`A20W6X!Ls_nvU`_x5KVJIb)Ih7)j8yq zURvvODJPpL7bEyF=BhFtCA$el;1|+Qw!3&$wntNt&N4QRv9tCU1RTv4KYZM%N+98jIn&Y14fqRIE z+X*CJPYsC5Op683$Cp=9^W}|bQ;y+IDYB_`e6wMRIM8} zJDq10w!!RHP=`21Jk;-?q!& zbnuBAIv)$-Z`9bXfSxJhNxQl8WitSp96?I9RCJI}-B^Y!0|SumaY&1v{-$E^6kgV! z)nMn-q9U?M#(Z~pV>fHr{~;gulg?21xt26LAqqldtT^#dUJS*b;O)wNax?q4sLkPj ztVge-r7dhrB-(zlGqkKKctLEJGR@S5QOeoy@wr2?>EdnZV9mSHPe-M~&p<%(M)dZ7 zgr#Bud?AzPs=-0}0;X>T*cvc}D{%!-u9{981M~<0GGC9-p}vV8qxW;My7xA8smo=4 zdusbOg^YqzD3~uy@IbBsTRkW0y9tVg`F3yTOtt)G)WqoM7MMP$tSsu}jdy8Rql%x` zINK`EV@^D6`PE%+OD`v4#_L!97p;e*&&rvjF|1u*9 zfKoTd!!CfHU9<(kA1wXsmV=*&D6Ysx#89lD?&|Qpqip1GW~GlB;Qzn{ph33)T2Ey3 zUsGC%T^!MZAW-ROc$18;x=btviZ%YH0ElwrykH~n8Z&I-KGm zRO6!!a;awL=sv3=i;=zk!4fnn;{tCUUncwRuA%Xro{EX!u}Y$)VfmqKD=XawGX>#4 zzp)%%B$dCI>Kakf~o{OiR%d>3)>+x2E%e49;t7ZjhX5x!Vhp%3vb z%nz6C{Jm5|boU|h6f_Pi6VH%Cxm13(bb-?zq>Fe!UfDk*p<08+(IQbG0B#x7emgHm zD9efQz1FlFjKLjoIP1%`X(uR0ntuSjjXT|B)LK7VrSHzYJ1gZI)C8rHLIVnc<<84H ziN!7|(;li|_Jd*vK%w?el){fS5C>lIoZ81SvhsX9oUkWtwv}@Sz4<7j2W|x~Y!hgN zq~HPIA~Hzd07|a=8C2ch8vaoH$ZxO>Wk6<1|FGnAJsAJp%L6BR{EzLdpQKP`vuip| zwTAVpQ9%|;%l(>W+9K@%dfD)93-bxi7sf$`1#!m;=;cqniIxzd&72aQ3=Z23f8fg< z*ZT`;JTU^>FL*vxvhU)(d+~ykt90i;@C?g~f@AVA&8okXdTF8w>GRez#82S=r{&%q z2(_hlK<{*b;|uG`}Zrc$aU z29PBw;~9GZ%X(w4wjIsPQG8Ct{uPGe@l&9E{__dC70?FgF!&Df>#}tdgBQyK|BvE2&}!mdV;h?OwH7fyh;he6l%AQ3%g zL<4gvPu70JDO#&$&^%5^N%`FT=uJd)y~~b-dSL|o8T1JyFb?F~U4=2f?i@IOw&IT6 zkWE=uq1W{7cUI8LF?^W21^gzC=02$KMCf-T_*I4v`f&1)%Blap25`%^XXJgP z)DH2N(-nO*(q-P$Jb&t(!?{rA1__(>hzaA|g=SlbRYTL2@bzwMF1i9b+gUr`k^+RU z9FAv;W6G&5MI?M@Xq7*hx2sLQReg>4w1nWw*(d&C`2U$c{C7xVcf`Xy{_C{S#Ls22e7abVxmOnaL`Nq(E8izhKVpjrMHN^HRIE&X2azJVWX%di<%ilmz<77+x3t%a|K=ZbSX64&DiZ!x0y^BK4ZuJ=t`sDMFgkwrVz{rkPT$P zBrdUJm~QKyKUCIfet2iYg9a>uoiF(?cejh*WUSEQ?AR^>oFn5xT z21EV8&pbnl-Qv&Il{I{^Q@smR`m`ZUY9|`vP8M(#T0SHeiN%8i8G<|;>bA>9&;pnK z+tp#tmm&2H8~*073L_eA7|VlYxgAUt;Si3^13!^F`SBeF@BNR@#Q!$x_>m2>^10tr z-%R5ZdDPb$nEn=jar|a<8_ZF-C&V&AY1B#=Y{CzlEY~P5+hwtnY>laq>4=mpotv}o5J;NqshMV z!n?o%j4Q_s@PIx)VFP;5PX4xkG5so%Fe6Bgqfw%ld{6*pns-R8z=zV>ed0)KgYpC_ z9kx)nJxrGl?J}352S68r3<_oJ*>ncre#GSy188x7LU>c~)0FwKq*6BtnNW&XdYnpn zHMK6@KF9>l_x#IgK8s$kGdAc5OzAC`Gby=x*nD#tc8GDsT7d3U+^E_^Xrw4eZ zfFHd2G$p#&l*VeBpGgsH*zSgT_KNccnf*io_A9XFnh{NKrvKz?{cjPcjQn?35S%*K zM+l}G5Now+GQIVE?LXu)#ZE{hA*#*)94Ue&sEKfx19x>z1Fg21-!W&K8!UC>8&qMO zpY`hy|}k1|vn9io^av=|p^MZKyt7UR_%L^>S_ z1UlK}^+vCA0l56MwcV{yXMeBDro9_EZi<~wm zM>^#i(*I>e;V@|Z5!`0YvuTNGJBszKJCcN`Ta=+McXQtWVHV`EJUbBTmDdP_A$!K_ z7=en)!sJnFIMgWeYtw!qUQ1t5VE1f|ix>BcIPipY#6T;>S#iwV2iSC`osr3iym zgsn99AUd;D`DEIW&c{tg5<#!nN?`rjNa|T8!{7?Ud}i3txzlP-2p}lXP2!_CYPZxI zMEm&!3z9?i>oUA0-{?B{_5XZXIljZpgtX{07<@%W$PPI{nJ<#Msq2}RhD(-d{_+@R zCL|ev-~uN$2IL5Wk#bZsL!SN-$%pKWvT3I7|1avkZF__KqpS!sq8|r@8LF-3LW4|C$ieCpBext${-z|0&`CH9-;6S% zi^XMQjo6bchwJUZM5sf|;$@u95R^9{1mFb-ZaO{~Y;rVQ*usVR&J>au+^yF? zMRuRo5J?qc0bsf?0_b|t|I)6Pl20y?zK>oLs%w8iVEfC)Z6tB9Vth3=a2%K3Al`sA zluOu}l*-kP!i!S_MzD9a{iN)kKy!o`K_`gyF z(8qDJa|Za3QE~G(+mXWu$_&pap&Vi~O)A^!|=5(7@c-ETLckTJPU4z0oe-l)EWY#dV;O^D*-dg^+7_5JyXLdeBJF{k-yjvGo=}QHF23_`58yAgL%_ zN-5nPi;94>DBUUDT?;57-5t_OcP}d-UDDm%-SJ<3-~Y_{&Uel{Gt3S%@a{bIUiWq1 z_jSPRGr|6G!Hq2L;k;ZLm1(HAhYI=$?sjr(2) zMyO#|s#QzW&%eu<^Be#EH|?;=#if0=ex?`W;cc5kh*47@+(8__L=(HQ(P5>w-i@$( zU+BTx(H>SYNLJvl@w0ed(S;l%8&@Uc*NE`Sz87zcZ^Qm}s)N7bK-k8RAjKBnJ}&>V z$MQRKm#)Z{erI-{x$7EwL}G#L)+jy}VS#;q&<>Sio`Y@qI9sD8G4u~{>p=@vFeP`FJ@$rdnCrL4l z?SGS>`l{vjI7GVsm3FxGDKZj97uQNaQO$TOu5ICycmb z**@RG;{G(rLY9tiO@~w4*+j3y!%YTB{d<&2&5KKC8z6E`NV3kO`|iAB(7_y8G#6xH z7#h7a-M}aMaJ9J%tXX*#&!3pzIHcfM9@e|}Y`wz97Cl6O(V5|eDa;YV$QEUXejfY3 zFH{q?{jx)tBt`j%$!^zEY;trL;3h^V$WQSPW(%-Iz_>z;+D)c0F9Sm-)XOEd(O#)2!7#mID?S!93{exxh-l!gj;uh;anE*wViFW zb-Cq#*!QZd8`)~MlDq~QTLB}9gg1GO}5_c~|hn+$_bObe=hwT{^5H zwfl|+K!z|Qav%8Zaj)F=elSCN*#VB?NfpoV9-R&4oKOPa$<{Qk%N;w$5c;h4OwZ1)u7XYFtq=Y|XNfqgF*J z2J(!e8xC|14zPb6$fUdyvX9-Qp>Ao%<9Ek$)rdUSIYFobzLhY>thsu<=>){c`s3Y@ z*MB7U{#%G#K?bSd2_oxZPU+kJHXignkTrGh6GW4J$A*w%K${8|T%Vf4XL&QelQ}q# zX>a>xkFd@6Vu`S)#@6{jTjO2pxz9B@#u{+eues55pzVc=BHx5PdY2mUC_W}4g_ zl=a)TFndB`@QHsu>X}K0XJEYl=z!@I=7)rK;kgt;Z{v_v+mhlPM%ru6>iIZbuXB)W&)qpu4`skrL{f*Z)Y&ZA&%(ApjT^ZRT=~$s&M=7;dbCz_ zJW5MRlO&{OS>oCR8V%VqX3Dlt0Q+*91>QON;qL6L$YLsswLEEpXDgW2E~=|}DS%A2 z`K=0zYXzD^1@%*@q<45Cr;*ZE`jglI#xG4kiy>G>`~8n2x1wA$?3BbHF_3x~ArB|mc(hsX?gUoh`#KuCWD9Si zWOlCk;y@I_*$Uc@A?lOpN2x*mY6xqT=S&dtCgc99m-1RcqkW?~!!j?~fzG8GLwbQWi`1WGKr^UB+Gm7QeP%st(H0islf%M?&E1&{OZ|{GG3(@CTAo`z8MaGCx)i&Fj$56_=)R?V8et}g1^H% z=YNx32L8*=MNXof?k;)%ZJ*RUdC(EEQC^LGGf3iD0vcIJJxYV?J5o8O+&hzhqzzM% z%G*p^IbPzUA{bwu#4tRVcNUZJSX)}(i)DbPe8Hot*bc+GDC3#WP`kBa2F9*;t`+ui z6IaP#2^|^gg<91Ry0PjKca-9Yaw>na+AEUrW@wevJ@pHI>ORgL5d`f(Fex@LP=9!2-dKHvL>kqccIjw#B$bv!jH+EB!Rku zXTAj!$o1g}_&dugq!~_t5;Sr=EZD^aPib zO{DOZd2x-xky(U%&&jU?82|1k%vw}+Mv~)4ZQYk&+LXZm9*b-jl1owcx=4oT@~Re; ze);`5*dyNAOmi~Z3Z`s$l+`E8X4XE${_LQ{YPn-8`YlP(!?%@~N5wnpg^Vj(5gxOJ*NXRH4V zVYw*wiJ*ckibx`hx8{-Rlr>*X%D=_3V1?+VjaD!gdA}i`bpYaA2RV9DKjh5882bl7 z+Bq(xINw}O6L~sE)SDi~h8pb<2#II8PDD^TSF?w$FV#?wb&B^3Ps)icvjPSOL@u-} zTB`I3`vE>IBGy=BE1dLyMD1YDsR=sEEJll7VvZ4D-|A=gYXcTPJG)u39Ph!7(-p=c zmU9hbr}igJNQM*a&JQ1`D?2T1*+qp=lbXFtPP~7j82nevjsAf3L1$>3F2wl&s^QX! zGbSNfIm+@Z3uXf78Aj2qTyq1%C zPVs`WI$Ik{;C9zQTGkbkXEi#0E>#4QRTMC}oE9^eP?C~n0kKWpfI#5)&vsW*n^z*C z3&z|wkt&PI2{*}M zA!VAtiZn_%mre3?0TfaLwViMH$#Lo=(qBtn^Yw`6rh+(lAzgS*9L2%{I-v1#LfD39 z!pJ)1>BM2Iw|~_{v2te*$uEm($ev$C-Yz+Hy%6m1FRBw1b`O)Q}~aW)UlkGm+)f%-oC zYRT@B_3`~+M1#w|wB)*BRXb+_B?*y&v6sxEYD70fQ+vz`w9MxvSFS{vQ^cR0IIsu{@Vs^ank!%&%RyG%SMmr~TND;`5Bm`p^+I43ovLk7`Oqi6JW+oBiXGOlXCye+ih<(c; zyO-Tk3%JB>f4K~)T@Wv6{k8%(9im}>?HNI81F&54vENApx`Y>Tj9)m-#CjcoA}2Q* z=Aeaesf-nZL9Hqa1}BdyaBVo=ERsm({we&ER+ORe1<;pU#BQ0S4QR=HWqi=m&Y{dl z8v=Nh@F)~NP<-ea#gdJ__=-tAS-3LZ(GH!Xq1+bOGO*sD+txFAs>g}ig^K@vHve~Q zb2dv47{N8XCnuqg)-&>I8yCT_7lNZDIhypf3u|#X!wtMgp_3~SLc~T0GjWCBCuW+z z95=r@N;wXVpNutm2VidZQE#K(DZXZx(gLi`JR-Zzn@6laqgHc_R)P?o@+rm7y*2-(Kr5&1vcW@G_e&&5A;KpxSJMHu2 zq3q+A^HPwKV(<2mPrF=wOr?xFARu;?aO~UXm*o5guC|CoT1+V4GgYa`tLHwribPbqvr{Ph$g2szOoB8RP30cY+X+iL z?kU{IW!5)HQVwpS{dD1||M9NzbXW#0?io^*kVm>OHMKTXgWkKra0PIR#lpwr#3y2x zW)oVJ(5njK`=D2hB^>Q&@0Xulv|xRBo*hKymu1)X!bsU4{ryuORt#(6lZXO4 zaKhC@?R=_5v$xm5o{)p-d|!%ShQh-BjJ9^~FXjB--cH+&qP;F3r=7^P^@?|2y7j75 zUq*$4q*8_x0{{374wp|Cpg!;2S@aj&ZlD@W7)c^813pI-J#;+P3q$&|(6}Y%VcY&1 zI?wMFb4aO?0n7pIJ=2lvK@`pc*_~um=IllBZzL}0YJdGgzn={=)>~Bzvz8nSq=CA~ zB!;u7rkx~IW-dPbB^Yd~Rm))xAMR1taDIl0fUB964!Vxv!S59`P?Myo-6H=|BlYRb z*ebp<`6OK@WNx-BKX4DO8vQ(*FKIJds}1EN^f}G{_W{&FsIw`wV*lK8@7O{_4n9`L z2w{YDIOrX@#~ztY;KzaEz%pGRg20gBKsxwwTq*aU&;7{>_2^=2IVosXulaaI?z+1c z+XTQUV$(VLp~~$^zK`sPJ|V4=L>iv*LV&smk6pu)^EH|XVNQi%Y-hP~PFTu7wvoLk zK|!@cneaVHD45kCa`5;D3HdB_u6k5WbeJQw#uAJ)%q@6ek=7&@t^!wHv&DIaEk^@zXk6qS1;$Zg zxwU_y7|Ir{6EgGfIguMQkW=5-PGlnR_T7*Hp+-!LmXfb_brP#Si84LIca)gE*Xhxo zT^Jgew(+c!PkB!W*0x~Ne7;y>I^9-ql*``xov;vh=7`jPI3_pOgZcMg5``+U5~+LP z?x~NnXti3~)$7^zdZgq9)cf3ZYAXfgY9MlUV(*cZ`B7mWbum)Qohk~!%`wq#YWY;SgUa z%&%si&5`*P(wN0zR;w!5GE#iS)|%#9vyA-+K6SR>ng+UVH@?maCXgW$PMuj9@%Dpv zbN1aLMG~FRGO;A#WJ?^OLQ|*T*?2eU$wO|nI2x`r>J`;K*We%WW*>)qL9`qk}3T=1;*_|mPwd`YmxsCGDAdlr?pXRp>Ao2TEP zPyGo5?5*th4wp2Ro<_L7KbgO1wqchFFFTvf0N2!tti4g_W-H_6A#T3`Yz^&x`h+rx z<#Z>DR|Na|>zc$Ldww5hv0!TPw!j>>g$lV~zdjS|GUCcQ{$}mmdT-Zk?~+95cucY|S>`iluip-+nBVeX1(LmI&6j+isIt(xE95 zO#>BxIslx1FS?J}Yr?^7o63lR^NV@~2RRfO*c*@(p!3~jUj|xmSlLa*xJVeK>&>^a z@(IBJ(~P_T80bL1$uaakqu3kQ9XC{o+T0AGO=DMjhw*;u;0d!pB+|h6p|Sogdbf}v z^gC|QAke?7%VnK+QWxPj$DvN_BdUIXS5s!hs*|z#$qQUFibrkt9OQm8+Fiik1;N^i z94P}>7s`pB!lUOPJzgAMn;f^1qy(Vdu&A!>aZQ^v?T=-C=RdnLhYt!pfEkMz*<1eZ z7CetLx&y%WAl}u;@~!jry8dVN)rat{Y_$iD+R2w*=-&(-q8bFGj< zO4u^>!^DulGA>GiM$=28F3FyNe+7c%2@9oOdln>mxIX)#;3_%2s{&UVww`?J%>iDs zlOCo2+BEi=S$sPQ4?Jf*z7iWS@yUF#vtLbqP003mlc8}S1x>GLn$4YF1&V?mfv(Lb z<-bmr*TypsvIB*0M%^4_FqkuC^7k@^T)J-m$~OTfvi^`QTy_X=IyWbSW(#s=DPx)h zz_jNI_Rho73~P9-5{Z9>E7h*LBt))}xJ_*V$ZhafV zIxTOTC|}tXq;jm@S>6>4529&pzpEaQ(25#g6i&c9>Piq5r(8?ekG}XgvI6ZMERzKz zv^&o3EZZ&7Lw8#IG{F?`1V z$prz=pgpwuiJAj854L&!%jPc5ZqXAn^L032`&YRqtFwQ>qOo~dC*CM=&wls zD}pj!Vf%MRiVZRPX|Vlf{~zWhQwH;8^%`}io0(IgLX-;@N2pPQl$i2(F}tf}6wT~K zUtixTdSNej$65mXq?|8j;S(#Cx>s;Xqw28Cv~J32;C8{MHRu}RdJS6LuHAB5`isp0 zchk~oz7bjQThaXs6Q?&!O`S51V=?K^3m{Agpzy&Y4s&zX#9Q#1NAC?DOizXNUInX-{3`yQZjbp z*Th*}+dpXi0vT7bCoqVxo`Hp%?u+{bpR0s*B=OM~FLVnX1~W^aBg;Y8r4`&ip|>2~ z{4h#8?K2gvbbGj2$Y2n+3Q0E`4d%XIqPqVnuoB9ZypSV3&cDNCMrqgnbSexJfr?aq zB+P<6r8ntWZWGD4Y-d2v?BYIFUr`e!&kRvYy;+s@;R~k^w$ai$yE^61+^;&xl#8Kz z>4mln+MLNUZLivGgXucON%LoQaZL)9^)!7wH{YTQg`VmG)n$~3D?35CJT z{hYGvHERzc0)OZ?>Se&+sSDAeGSt?di|eWo&Rg!1++Y`RJBp3Eh5Qzf5y~22E1P|e z4Nl>nRLc&zTU8>LZnW-7M|78&^md4Ff|2YhWdg}{L*^$Ni6$SZMN^L^(j^9wUaXKK zC8`jzxInluD63Km-4o@<5#|skH|nfL5ayzLG$&vO`mOIv=4tHqn?++2X#<6?*`XWX z@8$F&^B@_|gkI6d`sJ&6F1(R_@1)b19ai&BtjjOsY_t{@Zu75v&pQw?pM>T)Tb3&l z{X3yoUmu*YPhT!=^&hu=Tk~)Y;VW9X30>=P*>QT@H>doE`Af@7~YufRE5@8;e1c64Thx0ND#;9;_~MMV{@ zR(L4a?X?=Mz;m2MoM~(|>_}O{jJsV^zCqUxF3!65qE=poqx~T8A3V=RFc`RaxLJQO5 z%~~(E@v=B6acM5Ry6hB7XX$Q@HlSDcBX=5A^2$ncAvXI)76y+>oV+3)6q-K@Uzd}H zFkvOo_x!|m)SqE9TBx(A!-)Y;_tzP=F<%|CeZhpG@_$-SKGUDV8}cX2XpLqmB4{R{ zCe(Xzs7z^`TGGouGuUwl$szGT@v(HHLRw2 zE9^|+MR=PZ7ZKFyy48xsMotdn{ygt+SBSf_6+|-tg5BVq-XO(G7PfSGTV2vX#fFZaMs2R zvgPm&Rr_b!cpBGrm3lZcsmWFlcYL3xJ$_GrbNH*2?SM1ltl<%NxIZG-Q&)Av<6`wlXiE44$X|B>Bkl|9qt4RC3onUUCmfS;(5w z52QHOlO_jdCmF{W(#ME9GTDmWa}DwH74S+Oi(^s+tw68}Zt6Nktd}wW1`H_2qeJ&= zTkhmLf`R{~1+Z6-qmIGhiUZeJyP#4O?FL-mt&`1b@yVg9t)kS+8EsNiub1t)XJ7s^ zwrh>vavSu}ng6|&?Iq4(7WNQgm(Cp~DR%u2T$aZq2FdyUOS)Fe`LFW97u?Ud%%YH1 z5x-a%kCnk2B3Q90Aiwe=Wc+6@%6pL7tD{(IJ-0M)=w%cof>DdZ5iKs=4i@nYq)q>F z1*{Ub=HYw>NLeO9w2lG=81JNq+n5BMCpo0G5Il%8&)2{kuS5zFlLI-i~QO< zoG*5_wVrXmm0@4+w2q#SjuUrn2ecfkc8fY-pPIWVkTrvdZ;XzV1h+7=uZnNA9l^#c<%lBVA4YMGtn(V z>Baun*CrKy7cXB@9Rln7$i$SurMhWa+?~-f5jC1xzX+O4Muhii_EhrzOFXwO4W*Ec zn2cK^>zSMVMjM?1btdSYvVfvM88z;Z5CWe%Lqas<^Up!G(GizZS$sV#VsN6q_LFbH zEWH4$U@!kh#pr$8@Td@n=rgJx4((*7@js!wUf0bBUI%r}<4I^_Vo7bMD;mB+_uq1L zgi)r4zw4V}(4r@PkMcXdg%9=4bB%vh{?)ibGDH^9@EY1yVyb2&r2VWFCME@FnN zFs7bjy7#bmakn9wiQ1#4k%8(3-13goD{L|M8i`LmhjcrpM!kF1>4P0 z@r93H*mCHRQ;G%~ZUk}AqO2Bju>!!2l-J7I-p2K`8iHkwf14|PjY!Cw*s`ne(3Ql? zM{C*~lP1NlLLEkd3ZLQ|$ zKhAgNHbu6+o!Z~TjHkZ6IZ!8l3SqK6Gu-_#YBQ<{|93<)&uM)Sl!A+FbIn(~+;kNo zo2%L>>(gvfo{vd?Kw{+2_CzOy_Ikehv6fnK+Z`TlgwHnnlq#OP{od&D?eYD)21T`v%w9QiC0WF-3E463eNz#p2|MQ0Wm z{Oz79OduRazb(MR*krcKvqJu)T%rsO)z6>i@mV0zj8c>=I!%Bt209Ma?IPqPb3+`@ zUD08VJBUJ(p5YN;r*w**m)CRgF7^?6L%S&tB0_w4L?A`l3^>b&LpdYrccib&sr8I| zbQd(|kK?hCE(wTi`?(T&QHumN&m}Cb%n!U;Gd0Bis{ZqqZdeF80yJSQx!a_i=19KV zR-q=r-Lkf1zLu+g9GB-Y8kE%|JTI41;*IY`Z+1t#o5p>v+n{r;?Trg}``Fw=F*P|^ zc7q_$px3@b3SYXbLHu_}Eo`LvSxP4r+}_Lfa5Y9jr?=QIxYJ|)B)!OorL1Y!n5{Sd zWO_59<#ugWrLDHwsQ#8+e~ij=^BtA-e*Epex6oZiQc{v+$u`p?kJQGmr;i?KBEII3 zaigU(`40~e+OvHVjJrDc6eu_8a~uKbEt8Z5SCc5dg11wyuq@gDX$UeeB}{yoz;t$b zQuNR7b@Ij($~jRd<1!iluZ!SUoA5@mi4z^aQo}eupxxECTg22RDP$ zr4bRTcev>&YuC|-AUJ6Z*sUF0_T&jr*|aegMv&QFY7z^k8cnm^4#mP^O*jpC&CGm~ zf$6>audcryFDhx6aZ+fT!9T)GmNgZUz&s!KV^0tMrB@5KkYvT?u2%R0J4?@huLQtB$VY|dJu8X7rMch=eW?S8R0>%prZ z)T>{Fd~Pey4)i*a@n#{7QWXpo{w;a5B!R?kUJX52jV=vRQw;y`wCzSc9@j~E5lTvK zd_8yIBbI{BUD?eyzl$tM;!F{2n93Plwi8jsWfp*l-Y(&tw2Gz(Hi=JT#-_@RV^wxX z-R`~&B!6%yiNv{2i_OID$O!^QqViE>P;aOjzgBDuKf-0 ze>gV$Dd=cd>ZG@Q^t!*V1aY=al=IU{??QhdkNx)zOOz6a%h#PA3pewBZuUaB50m7m zc#=hq+6hcYpsu_))DAP_9ZW2tVAn*^KJFBH+Dfx8-n=+{71+7H!$pGf`(P8Q^RmFT zm7ExP4K)L%MXQQj5s9Rjv31Mj%)aHoHYZZl4c-)bXr)o*uq*B&fq)G7d^SM$Zd-N3 z+wIx?KbzBylE!^O52WAQy5L9u2^EfK>LW2PfH?#1le0_y91 zd6;998;*Jv-_qo)CDHA2z8CPA<=D%8bp6T9kZRkiQJN>+y?e$t{gsymhO<91h&kH* z(4Bc9FM755=Oj?{^ryPbNZ9GlxaI=aNus>VCivN*5QZ)rBSeJ@(g&_0XwbS*fEQQF z=MNIST_?l*OhzivCzeVu#KEn#S@aY1L0D}w@ihq;+p#~$=y-B5)nOn>PNjASwN3Jx zlY|~Q#ijK{J!FA5-F&XzgVnB^JS%#4#lpnOq&~Flj1xF>@rdlAiP>z1v5cc1q8`uP z_|#j@(;t3Gx^WUFFwn9sOD!%NwHgklqmjIoDHB66!A8 zT$g6*yD2WbJbCJzn~s#04#7*i7-%PE7kG}ZUs$4P%?*CcZJuhj zA49mIB@Zyy%%$uFTm*`La<{0l6nOW|(!YWRFJqd15It~+KJmf#LHRDFztmdBQ zcx8IrU4q8chR))bQui<3dtEdUeQ+VezekUOXnfp$ZSYvfQKwjo08*mpB-`uU?d$xp zBeH#s@8L4yA+ea~wnE4B|{zVD?`ClEs?n&~* z>X)yYXa7dh&#Sb;Z}V3qZ*^sLMT~#?iK*PF7xTA9|6HkdFJyqIiEHtxD;h22ZY3KB z(%{5bjG@SEi73xyIE_#Bo8>p>3!L1*@Dj45`$ADGPh5fXBZG+l3oGG8mre?Si|Q3a$qB7j{tj1aH^z z%8a<;6i;pzmTyk$NPu!}G{2@Oy6OkxVK4k?H&`7k6O9!#r|f}YZetgEgC$VO>oz@- zk$!_}rEsM-V{Q{P`*K`fe3mW;t+gnnd+WgGAuy*=>_x-~WJD*4`NCS{@^J$)Lh=TE z?HZr7UXQvZ=Kj=jQWrn(x3XN@C-a6csr{_5H*%ZFvz^LKJ^j*H)MI)ZpeAme&TOPE z`ATz;fIp=;t}QtdH}oB`m&7U2Kz7mnw+FWhQsZ*sTNfmI(%C;!Qa3Z8PWVBsfd_ft zHJhcWXH|Uo(D7xd(oyR+X8d9fd4pK~nKk|^$Xa|XAhUwpVqYgbzUc;Uh#GG#zuRaQ zm^$bUEa7h=vnsF=&TeISzwy3_gEtt7?e}K_X3Qdznexzbc4Hdu#O0R zSYo-|)sqI{EWoa726IT@wKB`emr^DOuMsX_TVJ@jl7Woe-mz$|A?%lsfG$BsZq)O1 z88Ky=k;Wn$!M_}YtpZ0P=9F3+N`G_jA3zaYR5+Z!nu=%1^v&()sZ#!G`b&S!!H?th z;wr~%4Vzi(Tixcn_ZKZbC#wu%^=@(~UnT!0^=(4N8=npqa49b1Fu-4W!M+ae>S8CF zd`V`T{Y$8JyeszH)9 zg5-;;6>P<^5~o1uO?)9!s<#guZb2j3xIXP2x{`GADU5|ULM>6DgzUq5exGJ@3Hjs@ z|6=yv>9@PcPd%!=$^vE21j;&;Z$I|A^`HD+fw7o%Gf3TM;zCCgymU{qi276T(8n~b z>y1ysU78Gw`B|F|TT=%#_(rJo#APpgP3dnj$|?p$Z+MaPqRbV)F2jNj$!ql1OJICQ z_u+bdB;$Iti{I+$48J~gKo)F zXaappm;j@@=O?7i{n=OI!y@KWT10&>=YWCs&|@UrG?a%kon(iEGqqDb4|gdMo~txh zG>nAm0qQUb@-r>u&Gb0R1o<=ZDA8V*;CwX2kYzQeOe=)W?Dk6Wql<(DU>2sg`?aE1*09UC0GaUXZcG| zIXoPD^^i`T`Zrs+_koA%dKp~LU9Ljw<_pRfucPO-h)*v0Y0f<*&((~lS$P@dpHd_p z=l3D`e7@reWA*Ud2-+OB$$Mtx8*tk5TblE#nyo%fRYyhVjB8vOCHlUgbz*KBNcKG7LgBo8a$w#3a zz#M9_@*{p@M=KFhvr-L08@t3JBRmCQ#^Jt2QMIS(rg3?lpj+dpUcWJwn&t~{TOABR zLG^1q&CILe1bD1xBm9-=vN4;7*-6r1YTC{lFH3q&4Ofc+gf*2jsIs__iz#BOk7UaRsXlSIWgNu!pMA+p z-VgZWds=;DX>5^FxK{^5(-g2=^SX_Su%Rs50dvB({f*1#T-3TjUzjmQRO4c*chmcO zqZOn8W|WU2DbB&m58!BOQ5*PU5==_oZQggK8BztPS<{( z4vyEE1FYQ}S$HBxURd1d1EB5s)Z@dP?C>7Z8o;E~bB*c5BYU}u! zTylU>)RcUA90Y}$G;sgaMW?KT#;^~_EKw4cu(Vj5S)WMw3 zlo_FWW&-k6FdkpD`ZQ4kvxx$D;U{X(!ojUM@4P8)IVuBA%SEuT=&tmhOyCldSSN@o zz8Yk<>h(l1UE|RVpl2Tt06xIC7+p2(9UCU5zDR1w4v%;L!_qnQTStKL`a)Lg9^FZxfvr zuTyuQs3NJqB6{3Sh|j$3u2Svz~78o<%Y&L)3Go&MA(@K$Lnf1wJ- zUZg1z2!K52pXO$kUut5|0sKpkKTOG$sr_>!3Ia)iKY!hfau!Ov_Sxcx>ZmK;bNFU~ zp)R9?~N;t=QzQw0ujJQEZgXS0#Hj* zCU<2|uL%-WNDd_Rw#O}q!S>5Tdow=gOnrD~zEp6KP|u*?bk(B&v;FOhS{yDeuBipr zB#RItLLt`r)#xU}EKe2WTl%r01es%Aar^x4|J=;7Az&})to2B;*;;0YvH-g=E>L6b zeUh?an5?8gx>3f40)a8v&;^lBWETYl{(Vz&+zi-)gb*U4Zf&Ieo?lU>8H;^zji9{U z%$dR1Kw8OQ&x-=AJ|_`KreL8mWQ}41I?;05+d>ZHFNd0b{F=MZw*@wW!l?;A87U>V zLfqJf9VklPzLGaxw8Wf@e<)2^{jD*F&X=%Xq*Eb-W*@U^L=TD=bu;> z?3#-8mIw#fMMkXI5jHY|ZQ^ESEba1zj3=l9Dcv^(|9zLCEU{#Kb8~Ap^sw<2R%}3+vRSp%%?|dd{T)$bl-h~CB+|C&~|dgbHO@5dHz1!$jl>U z_gw=;VyW)FtIl~vg0at<3;ZN41t~bxL{W5KN?!2)7-}4kp!mnH-^u^<$rI73*_rp( z+^Hf@#$yx)s9eE|z$~nmj4ra%DR-^k7dx`Y07Zt2It4K~C18sSO*rUR*Y5D4PP+*o zKJ(qsehyfr=59^<_S^uiJ2^CgcIjZ3#^vIRaVp4YgtI(4{*E#g&5CXcSzO%P+f)Vr z51?kx1$m}4v(@SjrT=!c{_BnSpF^hjIOjjfiw;U%aEoe!1%ZuLpN9)eZRc6LPGfdf za2@8Xy4lR{iN-@_e{4Xx6rv^vP&~q>!dp~0z^+{-=IQkPwRibkq-fHvx7@t$vSC#0vdMN!HZ|Y zz$~X_I{|_v)7iEZTdh>}IizkiQO=4oXovrD(LV)Qp$)G`0$riY#ibvNiv5=V!E=w8 z?tct6+|MJ0I+nkE86oiy2#gp0v1LNX!xVvTi#4jBrpodrL6U;Sgq5?dmz0a5sMPAG z%j_aLpW244nC91*i0A2^t}Vko8&KGVJ#5-6;{SOZV%TAE z+M+bgC9CEBtW=0dnda^N7R`N$!~{m|>|l=>c?0QJ_*oVH{pI8y`!)fa02z^dJ@V=d z5Tfc?&>)WLylNL%aLJ(Hx;{?TE9IR=$&v9 zH+AX~XE8>8TZAJ&fPcMA)H1nSbuvIRH+*|u+hTQpBKA;uRoHS}*knukOBY!u(Ek`x zn#9QQ25VZRl{Ovs2s<~<%FVJYNZWHJ~ zO)P&3Y}?daxK=sIrn-9_SifgwGeffbyRgKZe8#$QPh;??Yk6DyzWc@=KWwYEBojr9X;MXpA4!cSrG0CW6(*MClAyr4K{9vzoyy~XeQrB_ zxLq1X(vtgmXP6}~kSPvYANDU(*(II1;8?;4=kAF6w_DLNtlh`LA8THP^|Ce=)vvMG zOwHb(*G6cDH3Zw}U|ZL)#`b&Ay%Az}e|B9dc2l{oQrj>_TRK|yNJRDqfnT(1e~446 z{3O603RW3tdAK{^L+);mltX__jTIb^()q_LX_6he*n*8;q*ORsWUiMuBHUz8eyBEO zUG22o?>M1flF>uVfBn~vS?>fbE|fk?Bzm5?;lse5b1SvlZ|z(OQo};rJ%T}T(nj5Y zrMLCVcKx9gyB!8*?1cJ~P&);G0L%yxk5(3e@X!MC9{Nof{ps zb$0Uu-?pdd{<^{^_~9b`ucm7@>fuLje0}}`?lQmxE0g1^@0LT?E8sYIp{f<-51{>Z zS|WlEdFP(AxMP=nnF9zw*4Mqf$ZWP)FHqtRT1Q#mGdCZxvUW&GrQ$3{&NDV8*ro+p zg=s|1f!IfOh;5JroxWVw&zT2YmnE15890U%#gOrWf~4O_+px~>n+N3njfrVEE4Gu+s0QG{l{mSFChlkq1CT#)LKBkAE9v) zv=%&>2Y(%aeSW5P8zgktu{1NKz|sOu$F?7bkLl=Ij2_P^O&|tau2Nb^UKpjBG9za` z2UZ!~N`bngthdB}jhv&CYUZ$J2tL~4t2OvxD^|*5!lr+ni)-f)=)%FrGqAhr)xEE{ zLM~*2_!ppoDB>UC^`c^zua8n6vle(YD36m;B zwR^{EtpTfB{V1aCeYHlS)p@{r*c?hlrKmd9Ya62l$>?mR@foJGxuhEJS)Fw)31K!+ zGx|!fD2DI}AvjIqw+?}rr3P&j4K{mpj#~4|bn?$vjU*mCT!jmbU42@!gBuqfu;}6r zIWUj5ee$_lcf1!Uq~5=EQ(Y(`Pb7sn+CQ8sJjB+@BJ)}aJZ!Fiu+YJ*{-GoCLM(Xq zYRtsco0C8t2XVT_2lMxjXPYSiJneH8Rfz=1LYmvU$@F;_4HM1ig71BG?vE4rG88m< zknT1M>t$`YnjFSOB*l*x`4qXHK8C9yS^)5+gX6xk1QjSxO=0Udt-zCI0`a(kq~@4> zp?h|5N9t;e@vw!mTW8;VKq=&?#;TNUc_34}M&AvDQclV$eLt2`d>3x_SpzFs5j>Bw zdCCfeSooHCT5F=Cw9UY%?xj6q>i1S#r1U(uT3?k(GpZ)9igIV5QQa83e&aciVqh}imc;DXcrOLl)TVMqpI`;8k=FsQzO}efy8*Vq^ zJPVY!x9-&-*$De|?KLg0nclK(`&L1TrDcXT_maPM4$F>$Xp zwLsH-@+pdY%WF%6{nAb&wvO{o61Wng6=+_ejS_3ko47tM4>(h=`pR39@9iBK6}Jdy9qJj@y>S z2$h+FghFM`ydRIVpFrlw#8P|aBsAe|n0Ub#Cx13-wjRQ|Uok^Rs{eU;&yi*2hY%dDCa2O`+Hb31knW^9eEy=w6IDzP+x;;`R7{4eSv{Q@E*<Dk?^%Y5Cr@zGYGx0Qz3$>dwvD0t+j&GKbx-V&8(7& z>pYFbTzog+otFNq)RoXIRu3ohQwBLq{u`!@1&lS4)rc;=@`DOr+bTaoCX>93!-x;C zv}5$IF(!w>zN`(T*j{B8mmlLm)7I>Xe$nU|)P3800)8D&Mc6-+drr1bRTY*v;yT!b zp1B(eS*zrn=k3}~DqGzbi!NQ_b(*sXd^xk7j5qytV~^cYc{Bf<>%n(t8~p%?X_KNT z2yqr-YXi@XrBp=c7PMisX*d&SGT%q$V&Dx~_Y$OK$pu05Hm<6JBaU^W-AsVHwL3Gd z7$#5U4_DD#;toihr}GeyF;`TwuMsVhd#7kI89M@`yR0u?g1_v$~xBi zn_%jarVf-)H0%%oLY$8jTp8SXcj3Yol}2yVWzvEs9O<;%n~^gEUCP zFo-mQsFZXGNOuklA|>5j3WCx|cgG+t-Q6{Sz);Wdzwc+g>wVufU$3?1!?mxy&wZT7 z@jDe8wTA9DrUcN*I&AWr003C~v7z!ouHqq6+^0BmqxV)_6T*HjMw~VYhn3RwbgPC; zu_WvN8t zuByOZARa`AO~Pmz!AucY`DQq^F9^-35;Sfe?UpJ&h^mJ*@zcPb@_HnnB^RF|7phTQ zt9V4Ck^t`P?@W#$W?&sDH6p7>+~Qqhh)oir`DlIOj-T7qig6o#@V>k1DdmYUNsu+6DA=t~XuSLq}xS^k$!*j6aRk;A{3__Fr0d*M|0L zkbei**O@tJ;x6Ggm4vgwUO|@i45#yjY2?MKWT~P$p(EM z8dU$$tCrsP%SY@H0b`L92S*v<;mkJH?O#W-5zn<8rpw(?MwG$EI;-N4*yrBx%I||K zw=V|Q67B4V-UScn1Gm;!!p|!O_{p8bteS)R1eeY}6L9ys4jT-ozKAF}I)%bZBa6@<~6z#q7M~OQ^j?xp*cbK=R~qF*yRHEDc=Jk z;=I030gp;+n$L=-pJDOJ&~L}rs%t+)S}mv)^Jqef_#pt(yJVM+vsh@(I|!y*wvlWa z**80^f=jsZnu_prn-U$!U1wnISdG-y4AZ?|OoSqUT3J|>5P|^jC|zSBVyw0R`>l>q z1>z-)T({&f9v}%wf8YWT$V)JTNSuSFrp=S|k-9Ra{VsVAUI6ic&%y_`-nSO?i4Sui z!ir4e7G%3^Tm&S;yZz%thq3&$>9Fejf1^G}!34m{e7Z|OMMRl;cAO31nZG>};6f|Q ziPPWb2l&~Rq<#qT{?Xhq<)8jVgMChZPzg6|EAeiv2M@wZp zDN6_yP*gh|3Eq_lW2r`Bg)&AOZ`w1pGOf}3@T>}^04v$Qqtz_{1xv*hU3#r6FixUZ zVmrUWas^Ac|CT#`KJ6x6!1}<`bUKG6%*TzSU$;GGtekr*jE5`gp%I{rpD>ngb?X&a;`ceU^39_y{YjcGd3BgA>Vnboz6 zEx-%z88#L^g=Oq~H`C^WtNSQOw^;QNa*ybk$ro#98pd$zqU_TnyC&lfdcGuIC zt7fNg(cG<*EsB2Ty55Sv4!iBu8GXb;#3EVbu;?mt&)kCiXBuH1Qd9advB4( z96xWJ5s_F&WffSl0RI^y;$i>5+(%kE!?$yr(8+6leWZI6Whi!BVjo2oFfwd6;N+dPR4B>XU z%y4@o-}exUCfFV}QP`?~4SZ1srIx=gWmqCy;%M#o=MC!_Ao0Vx!0r5^HpZR(A*ylNl?=E) z5V$g5!h3MmCnfpi8}+7CmztHW{@g($lNu_IbEcQKrNuxf>Nio2?Wk5>CPPif%-3s3 zKZoeIvD|@08-T;ww|Rqc68wzlJC$uPd1*wTb%go|Mvr)<+Dn!4zHw<2m@@|PHXN-vI*=c!|TWLCXL+h2JG+KFpKGR`Z~ z93_|h7&?7@A@KEZp+!k5!z?Y6jI5RFP zM7PBu={<`$V}iPVp`n*L+-mZmkoEkB^wZ7^Trz7a%mK9qSg(?!zqN8r;Ga;>a6+;V z2xMLUDN*n?vJU>=24C}6>a zL<=AAH12Tf+v`mbzCTW8)jZf`{Q-b@_WN-Ifdt=(d4y*nHWv0}bc`1^siOVk*rBbg zzcn(VNH1dA*5MHrr?;O?GdX0myZd;a;fzMhz_{MLtUDqwuT#m!F8ltvU97&~VfTu4 zufVjX0ZM^e86O&d4v>y%ZqD+-xnzqEi~$(0vyw_8;0qCVnahP&Gdb0(26au6EBj*J zLws>Xmg|(TmvY+471_8imhZyOFcvjcIUI$*>(y+p^O z!(r+*N~wdtN{9D43R={Ir-_Hf^!fE3!4qVZn|b(a))>cEn&YzCd;JZky$x48@6$-Q zNsBJFa6@`N)2EDr_v1;0VxXbQH1!^U3x&|inA=eH$3`6<^K(s3w zM(z{hE8!w0`Vis$WO19A;3JyYivH#)H4Xn_ja?-OMcVrN0(}J6o9?n&##S?-`gZt8 z+1(sQKa>MH8Nj*42(12uMWq;LSP1B+>B#Y0ZfVST1&k~HBjyq}6d(?|!Kj-819h93 zX$fB(#BhBl{}afI>>%Xn&8K-h0|DacWw9-X%~v1i9ee46*DP>%jmsIsLle*cS4rm- z1S7;*j5$AXE0So;cIv=v)2ge}_JG#vKYwFc^3mIaocFJPFFfa}of+!CV=2%-PKn1! zVN2s3`wm|{E9tXB*mry4)dnTLy&F9r1sh?*i0PCOv7+x}d{x$7Z@pf6om}_C@|@jZ zeeyA$cK7x(yb)Z3U?#y4TZCg;d&OItfOBHG=_{-w4_qoxda@5F3d<$rA}FKJNm~p; zKyrSJ+iN^x^tOGhKt8Mzp?n?Xlgg_73gEURdm+)yh|yd)Jeatb|40E85oVLCA$m(y zl?b(R1!Bxhrxbd>qQHP$tq(HtR{%ZP%*uYZ*-ln3;01egi_)f0D?WKXYx_5W(LJ$M z9e}O=5Tl`+Sj0EBs3`Lh4QLP^-Lt3eon&-R;9L49ZBDA>%gEnC6lbd{C8jYf?J*oM z4KmsWQsb4T!1QARZLbc|rBC#ZRg28lX36O>=gmgso?23lELStK)fJhATIl(c;i%Cc zs{qFtlt@!fNfo8%dtW14MCr=oVBgZc*~*7^K%}s8eP6>!OK(<36J5Q6-_zXv-LyA| z;A5Md%`NZmSrz0Y02fuB^$5L@OW1@P<&z-2QN~lHI?9&~;G=I)pQvW37T$A`OOC^c zdfeVIu=(OEwSUXrSru`zH%%F=JD+`&!npAzX+xZfe8wn>!2cG!4?kk2+}bYO z-JWks%Q|!$JspwcGyZDvzdiG^Csao2rcPi^kdhCp4h4`*B}5%TSt6sBV9I<>rc`93 zax}w~0Wew{sGbjk523UwFKRH2+HYO~=q z+bCOug+59!K-L*d)0mXscUHYpL}K9qS=fbJ_mSAIIOL*6j*M;JDsdLbK?Ze*3h2Wa| zz6G8^VX?V98kT}|KJ^2X?KejAxoTHyoGIOcjn-UC#(cSU_4Xwc;A4~0dD3=s+*a2JudqHY6Z>JjJ;Qbt|{R5u|tEIi^>Sry~UgU`T5lFG95A_#PNiK z4&nVJ*P|EeD`5DOFfGzJ&O!2Bz+OMpIXW0e53`j4{}h&LLOA0=Pr@U$@~KY8m0MD^ z-Wy_csh;Atl$Sn70krAjKD1c13k)vt0i8SliJ%o5h(M}=55g)R&JqMy2Ml|n6VOmWwXSJQGQ;usqOt=bu9EYl<2%3$Sc!Pw`ZCT65aN{fW~K_b z1Umh4IPPYH#-}SOw;-Z><`HaZ>iSu z5b%@;{vIhe%XaHSW1t!R?~6gDLM}C8zbpaIriWPx?A%5nZp&K)oiWzIUJL>H{1W(O z_@eY3YeC`mab|FBG)&T`QN(tCGjkdD0XbtFfV~3LS^iqr8~6fA;!wT2YE0^R#7~Ob zE0_dyQ9vvg{SIVbcO^IMK-cLHnO^`#j=ag212H?Ca0my%YN?aBj|9gMp#AW|H8*HGfOD&83Vh_8Zv$FPH4JrI;{3R-He5=d>eh@s!(US&W*d+;q;Gmq@Tf0KN zILNxto%DpTh`Cj2xO8T(6(Gr6V#eVVw0@&l()XrPUCgN2tSzo_%MZ}KMkXcKcjzkVChaKvuE4}OKl=V)-Y~-FmmU5k7|@iIxupAkfa$CE6Ne^#f<_;F;EHpcr0Dw0Y`IaMxth<7n{T z)=hQSrxQBnRPd6sbpb0HqRtwYBdhc-i!We3pkjhQvj}esHZkK|+3(Y;M~RaNO_=PY z=BLA8Z-A3D6zLW!<@ONA$cC>?(h0`_s0x(9dV+mxsmm-pB6jMCeZSvjay|@%6j>O z0;C(-6W1p*0loTkUgG;DcPp_hO}w@Yw5U9ID&y}X+>dfCe2QN5w8tel?4-XVYgwWD z^_X#ov6a18v5QW8LwbfcajVc*Sq1hMs0N+qSG;w5B#ULdYbm!^g?0LjF;e?Ar+#6= z7kE!-Wl7(YCs-1o40a|kr(AvPf{(&oiNRU_ju3K;wFGZAAH%ma1=Zt^grW7;HG_s- z;iLPMKR(yOMKwAwp-3#SOA#dFUD^w;!Ud`<-JA*055$!Wdow3*DXJGs@9u3BLN*Y~ zyT~H4SUcWh;KmGt+kxk%&?0){nV$~s26Gpn0jrK#>wgT-ltj>|(;{-Ll%fxNaT zLd%aPq^*|X;a8`;8T~5`CuWkaH)aveQn&)& zg|5`hj+NUg03J#eYx9N&NXpa3+Q zN(+36$hB=Lg>eCtwZB($Ej6Ib+g_$6ML8y>=eXCe#Bc+2nhNFXr5gyM4R6LMP#eD5 zMDlu4uQe0cJf57hN*4Ch?Q44tdgMgZFE9YD3>7tzGnIj9C1{qOB0ot+`;&S0?1)Zs z0n}5A^Dg~SIp=I1im`62Q_hZP{UsAl%|E%QSBvUF9HqhQ{(^=7*t39*&bCI5g*G*2 znwdzPB7Wb=O!2>mylL}U+p{Am#s@47BXg6h+Lj#*e(Om zZFfneALA{aBpl0$oL3U;2S6V=OuWR`H(th7d%3XY7atks0rE-o3u0kDTW)AxpqKP{ zVfRK$fm+@d>7jpDzQmpp?0)SYNALM^^#ZCmy&k}wSGWR-xY%eB65-4eWjwjQeAl04 zfqhnWzg6Kl7^z++O|Zp>NhA7#A#xq=Lv2nk!97;jCTw4D;&Yqty-j7ERJ`RM&tN>{ z2#PIoY)X;6yO0|JqV}0W$tj#~=1Xrbx(c#6m^LzTbwo97+Klk;~_ zVqQwAE@hrAi!}nk^u%)clo_A8E{pww_4z~d59@-yg;K{!f0GVdO^n!}ak~6L@*b*D zRab9)DnfEq+M}(-UZy1Y?^(LT@0a(nOqv^=PrPA|$s0`OFQ8Cx5a~}tm92N1{3PEW zZ!~4c!xb&Yg5NFOr|uK~9?H62re1HyF-@a5LHD1ELJlS*#n|SC^H`~H2Uz~zyKql( z2V3W}sc=FlNg5x9Ggpb+KZx-yu;fK5_U6t;V%6j03~P7u**JBIIkUVTt!F7eZVoLe zCo6mVY;G|56=f1VshbN&e+$QBBhw173opeBVmlHTezqp`gr1dF3`iA{mMLT>z@GoT zzs&gTZ{(z((P#ID*4&b#)dL(u#1|By3vZj7ZDS~6n%TkfEg@a3yAiel!21Yqs>>Aw z1)7GN0||EWC9P-^gtrSHHJY_qw$seE+)NtwfoW1XF-c2}*7Vp@?**-z$=4E;awB9b zz@-q4G>@jj1N%%pIc%MSc(S4Jf$=o-=w|G@8&2P7V!$sE|90T*QVd9eC0*jV^@D8L z501E%AaNcg=+onD$pzNmqwnO+p72$71L55;ZI1Iu@~v}v)`Xq@_(tbw!{L`b-!amR=aJWXnDEMal)vt+ql- zI~-!r29{5gSmkzj@-q&@g%SlD2TI{T=^$%tvG|ZoF54D|C!-6{^h@&y<$;!M&I&{t zcXCBAZl%5t_KPj3ZP;j(#iFd@bN%*bCRpNPYFTo_?K7TYX60{NeA@nY_9XN>wUwjh zf5rO47U&;y=XowKdrC6`1dX6@VSG;e7Lq&AAEx{@pU2;nd*Ng;x|)k-+`FG69Y%k% z$>om}B42;}R$RTBCq>1q*pd-kZO%)*Y58ukV$3+lo*}X!SBL^GJ(o6dDW)iD=1i4w zbodx_Z@soT@M#5t*QC37abI(Q1T&HT37tSCIdH*uXY#$REx)ruLsr@P?kq^)4m5E0 zgN4d+&}f9U1$C(iyJG+dA89lMphYO>h+e5VQoeZO0RcS;mR3oZ6kghT%ft~nRLE6& zpMgl#YTD889QM6f4i@%1clS#I{}9`_XE^-FVvv~fZX+<5lJGce!kyLAq|Ex{*&v!t zd43$W$g1yTJ1XoF>z}CjITWP-O8(|3SgJRXfh8H2=M<;uahNR2ld7S^4aX*3(XwZv zMisA;0BS0!_tL<|h0kkQvH!~gSdAJog?O3fJ+`JY*X6F9cbk1UfPDJk&5dU&cChQp zFv9r}!VQ+ntvrX)H+mS2H*(b-v0T!YPSk4d^B~$p(b5J#8%T;hrP~eGGk)-K&cvrs z1x<#n2U5CMABOI_`YN6yD1_I=ixh6oxxeFD&@m31)V*nFKCJ_Ykvbz@G(HaXT+={@2^)I)uf*{ zK%|jhYleQW|KLEcdD0srj>Qip@S+9+W8~hYh@*Rgt1#ydJZ~g#%Zm0pqZSE&W5qkl82sZbH6q<27z&M zBvP965YWbyDw!r}i$Yv$cElJ`YB*4ppReWLjCJfmog3gnPNe>GWVX0oTvJMA#i31E zuhdkT#%;*=%>RmILWoTPUCFT~EkBEZ92HnwcTSkpCcr(WoAxag>C78%h7AKUwjC%x z_nA-p`bKKBwXdnVeX7@oOF$I-fnPsAFie=m zhpdnl>)z6Lr5uvaXzGvt6fgU);JoL!N5V9oaMkmVVEfC;wY!4KU`dWuo8TDQ`gPn5 zd1vPgx8#yLPIT{^wJHu3&nV*8m5R30*VZ|?TxzkK#@owkiDxedg99@T;n!~u9Bpfq zGdI&dHd;WH+}(79CW-H`quP*jYGMR!^r2heALr7Z;M3N7X_uPC9@h1X}YWt#=M|tzU-Jsx=tcq!bgBL zrT*3q0q}>DT@#11e2eW&k%yhg4m!Pj{IZS1-hVU_j*Cl!LP?kixDW0G1}X}?e*Zxa z(z&+5^3&OFj3sT3Ul&?+IC)nX0OI^%bL?_d+Wj@${7H7gc$8aV)2I&aF0Lnp6wQ*5 zV?l_&^<;nAc%&e*U`1jxc+jiV`ZL^1=VdcnVs7y zi1~oLFIp4+_*(^#c87<_l8&*^qkU2VC6zrGm(X?9C5IGWcW9X{VDhHK`=Dmk*`9oW z3=bo*Enc1AICLG(2R2GYO08xlnWr<^TrMz8p1d|4J!g3+rAVRqa_UUSx0|gyAv`=> zHtFkxMG9GAII>cIM!JVN)Kh|t3Z_mvw!eQBmJqDCiu0n-Pvj+z1M9j$%hA)${hs`1 zp8_2a%+IphjK`7FZhRSdWgY*B5W!p@?k6%n!@XyU7I^h*ZyTuc!x##gi^Fw9!Vm)vUpAmLF*YPNl zGH-L-uh5IOM;~v?v4BKC>!Tw(_p8FnhkGTS9NHRxQX+QuH!+@0kAEpY7^T*h0>H3m zcfK$*CZ+RJbiZ;eH5e0EnGPm)xSMkx8aT5yKfPyKv?>`lJ^jYg2mvDssz2Tja;$6i z`RQ4h``c%#r&{hhPitWo_nim;Q~ngE zZc~zP1?U<8P4}l{lgbhYajmpi09`6eSN$#}4n&jQ2Zf&cvq9EG5zOsfqAP4CW3D`z zJv!u*E1}*PPEYae?vo=tg*5{}6RMp*s_Ot_=1eI%ZqD%e?NjMFjWHM}{t-}AJZb3G zN-H%*!e#AAKFxMwiQ6cl&;|7yXoYt1O zyv8(^=e)1HS;?M>3--`>+t?X$Quw^->=Cat2fv|_osHbJT%F-TEw;beqBxfsdZj12 z6p~d$_Etjk2JX;HX0w_ZNQ0hSX%omi4kz3>sG2kwkfB8nf5eb6b$hU^U-6%*|9{@a z|9va1MECxBJky>1^)4WcAsRsmApJyU;9-nQdvDdXDEt6Oh>{v$3LE*tX_lQR$TiFW z!*ABX+6!@x=Z8JI(+9T_->p(DeY`f@-BXK}_@jmdm)DVm?aVg@SsRhD-Jv_An1guWUAf{Sk1p_zve&jBi{>R~pUc|7Zx4*Qa?X zkHT2I8`3tZuNtX>TJ+Dz4X1rD&e=tqm}|DI8XSQY9;H4ir2XsxKPd6xCOJ%95HL#t4w;5CB z7=H&MG&n&&E4%|sZb=W&SdHw}u15AZd+PYK-0a&A5#ZlfKxy)lJL(UO-05}Y^0e;Z zd;kBRQI!)WUsG7Mk>pwDUyb-M>pCQU*my1Ls`;&m;2x{65y<0XjEUz607S61F7*4y zr2(X0#R1${OA4E9O@hFnTb@Pt46MP74FFLMDnsf9I~tWQZjE|#OY6#YX!~iALxwQ( z>0uq&Ie79QFC_TJKA854^5LtLbYKLhecg{E;;*|*bfIx)Wzm{`rC#HO70jhPL|4y` zF|o+#VtkOxr=-$b`&6?0u8?X!ObqOl(YhQL*6n#G4@Hf%-JCaWR@&HZ;}$yKDk(!j z)4)3EY7xmj<44x2Qa7jgE4c3xw^T}-cJ&WG-R>{mavhR-?7YiLo$Y@3F3rSBIcr*j z$X30&QuJ&)y>v`ezsPxC=S$RX9py84Gu3sJU)PQ0q2oW*pFAtR=`!cIWgM@Yz5J^& zJc&1M<0wq<;)iDk<-)K8Cja+mVu`cYl#jmR&HtLg?(cK|@85UPH_~$>*FV*gLO=i@ z?JiIij)B8ia;s|ARFfPwp4|L4Co7A-8-FZPmN4bFriVq!Jhcqf2KACp0)5x+oM9o} zZ0#aFcRuBvQZc?MHj_=L%Yxmd3|=4-V2QeOiQI$_Ef5GlNwEkuZ0!KJJyHb>x{i_B zm5~44-;}z$+=yV!oVsSvIhrwP!dR?7xlOO%UNSm~lAPrWgMF`8X@&1upIAu9)|o0| z7rLbt`{>eB)9P3>Ti-F^2;=M*J4v?=8)B|`Q1kk>7F63L1gw#sc_Nt@M@_bi$J|MN$U;r5f$0!Kt2I!Yuo`r1 zHtdhGMkMqshld|p|N4VM`r4}928r2c-SVy<(2ANVPi@HYY-lc5IJ#OcOYHDn;w7@H zpmu%VV*3iGTSVXdhcO3&*Pn|tzAmLb3v^)&4%YX0lTF*J zhQo62llXjzyk!=8*erlRss9DF6Cl{pfM6;aBp?xX)fz=5v_41CC3-jA>gP1QiIgnD z&`68??H?oJ#6Gu40wV9W&0VpE*>9ZjlMcyZ3|V#LuC@{%S?$LC@l%OBsfF2it2W?zaTXaO=pf@ zC=u234VtHeGNSD^cHc)9<&RC%@w@nYJBIZ1oc8XG#nM`kE@2>erWLZ>1oPDHbLA@E z4E`sCs*(XB&Ai{3gaNNj*1*dcUB;C;2&aniOh{4ng#tp;>nbF%m zHQ(4P;EiyFcelrm+XEMEC4S8wfu-*mTD#USB}$+^ty8MJt{WZAlzCBrGkWNJf?7*e zGTi}F#U$$@hv1*Xu^!EqxQmB2=jChMfEP%il0n9~f)k&0llSR*eOAH3O8o!wc;M`m z8aCx!qybv-5h|CKXD?uLc^lK&bp1;v<)=~e;fv^rnhFCP4{lQ6*KMEr#czQfwc36T zNlbTfoI(*uMkrruYD!8UF3FmGaHY3rRfE8sNL3q0N$t1kJ&1eqvr_r-j<-;Z+5N3k z!;b{ki%n+h+53I@a@b!AQ&`!lN}l~C+_)Jb8{ceg1=v0;!a{ILN47O#LkJO(E*V&* z53(+(ji2vcIT}&>KAm8oC{lC>qL}j*E zr|yFMF>NWPBczte>A8Vcb+3QdqGC^%@W4f&oS(veZ)+9P(sbMyc$kp;HTDP@_w`Fm8n7q6ZRhNXs8KnV zUlouxcGh)L-+4ayPhD`OC@nQ;@WWcjov~+wYFfIhrXa6R8uCR1WL z_EQk^!{ny=)Pd$eDNM?zSPW>=%71qT(=+yo0;GiaVyxZ0@T<3xQ3%L==ujXeO`syv zw|gA?#w-jw7mkTw(4@)04zrjGP1Z~sn~#lbD&IU`we1lM-K$!hkG=&Ja~hp4pVx{V zb|fjTeN*$nj6QWWU>jzMu1GFTGz%Z#TH+b|{O5(`Pfwu;2{n$$u2HG(9O&q7V{#{A zV+Om-AO$edqDl*_!ILvU3Hy#--*aB5$yYgEw(oocG52_wi&a)E z?YB(5En(iqk{KNp#ve$NRJTeA2_x|I=)lB1^Bg8>#`clNt^3$rl&no3_ z?lO{qJi8mfOr>=}w-%;%fB?DJ(taB|{Yk(Kg9zhGx60Hir@L6M%8( zBdc3?hri20Z|NGkeVOL~K9`1@GqgVvuUWMXPSX_I_YnSj2j6K_o4!KIX4U#;>aoQi zKm5I-XXq-)m3g+@^)^t4^XB&y!0u?%tJt@5Zt^8n_St=)ZtGv*;6XU)QL?UD)=*NGvU0l2_ z6(&K-m37_$cbR$LM^Q_N`d(KpGgjEVLl04QgfpVSamc!r+VLo=t^GPh-j$1<4$GYI*<2aNoPKS@M&V3-Acf_Bzm62r!JB5)iSpsG9W!ns4?yb2Rpr~h z(8tz3brc{ax(CK+MSI!%Q(fFMN~L>m{>_%scND>t7DsS(s)a>=H@aT5a?Y)jJEr}Rx_cY zEOYBOsQN;+dsJg0nXAsFBSqQ!zX?fsO5QWu1!0fhXK7xCJw2>MX|1MUu7l>g1lbv? z4>S!pDPwOR9Q3FHaD@d3upPGroOd6_sI`>_W^z?uK>D)m>$h=NHFG}K5+r;@--=OX zt1R3cxC@th&s&dG@4rsNU~#4~0U-DiILMG3iK{LmCKI4DN%2SFZoV*kU#v)uGm@rUu8QS3oh?2R9qo9RVIAK5&`Q|m0KFk<2wb}r!><7L-+MVGu zCW715N~Dpl{peGUKtC>Sl8X!9`CWs_IiAaJ_@3NnRQj8EM{a0Q9bdjrLoeEF#C|I* z*|q&!k;=XEX|lR%)w{)06J3x053BPVbJsGn{^;xcAKVgy{lo<$a!4h&>7WfA>%i=w zE)I?lv--s6xR){_R{WNZky5f=%6^~vIXQHo4;Z;ih@vz2Bk4@d;2l0b=H37Hobs_S z*|~Kx!$#7U_NbB<{@8Oky-DYnD8gN7Pjwax-O2kX?Pl=`YFBDr@@seAysy@W?f_Kb z(Aq74H=;8z%Wd5g8i#mD8}U;AEQU6h_AEQ9D%#eU?`tj(wa?0AyxKcgt=NnJC>^t{ z9IG^HUH?Y7u6!WP8uFKQA%u0X!)(YIm%GR-A#9yB%=UVV!7A?cdW-VRN@)}h+bV_V zl+Ve4vq7DggJInqr3TG*IOfyw3IEknaMHrc(VoKgO=NSlq6x#S0i@h<6jo2G&ina4rOucLx0T&@AaoSc=R(f(~J@;TupHA zIenzcNxwdRUL(d(bA1$)HCG~1_u!bufbEb8o*@w)d#qrucSSu3kvmqTsaG(RTNLm~ zby=8uy699ZT<>lG5iRn>#V`cuG7rYn3! ze&TMs#f{d8TbOn(un~j%=?xP)h5hfjY~_Ry2Z@0d`kx~s%jrys<7|ckRKC7W^vCM1 zSOAFt{a?pRE@FsP{A>>hspZ`)`w2dc{{{3%O|9LJ9j0Tj|uDoQ|T zF%k_2KYME5XqYMAm+5n~8z#SFU5L#TM0bOH@yptXk%EAZoAa(0>zPKT zy6bpCu;ba}nW|^h&>PzwyU7nZ49bV^QlZK86UZ!%3hSp#w~}vNrqB>~^Tmpu&?47JW-ru;Bt$q|ZW}-#J;Nm0 z_|eW$_6nSwNT7W(F15O&oVhLL%0xRuVnns<+N;$Zf&RsIAx$R~&{ zU`D*W9c}CGL^KW-4oY^U4$qv!lLgaUtpXS&EGvpYQry z)vNP!H$nHcYSs6Ro@6-z<*%d7>uqIn@K@~E1nV;U+q5?S^(n>vL^mv)m25;f8*v9x z8wE+@uJZzp5uUIJQKNvD1Btbhb%S2quV4F?i^50|ryx(jMy{#_9i^fw!ca^EKaEQL zHcE+BMx2$4$U*X`lNHK$^@n;x%&5Ho&1xxCE09Eu)?)5cVKIs!vF&LJbAwJZL56%! z%#D&2W)_+hV8g)BFdA~*CLAXk7jlkKMp9ZZr3!rnPxXIkK1?=;6pVNPP#2?cFiR&+ zOz)<&jTVIQF1>05)A4Sby+*djKtOMXje+FxmkB|)#vkog44}L9A0=^L0N$3Hd?P!= zFqk2qvHs=uIsMHUiV3n_YXZ<-3yW%LE)mfIwV$nHUptNf_KqOY*t{a)_xs}`05Am7 zyaa5D`rGy~G)Wp|c0lx3gSM5+81Y0J2#8%%7Jm`81`&Lok-352!gqi`ag4;9pZJ?}rMMRRmi zm*&ZK>qsVr&F+h7NiDy`zN6Yf6n(N^qZ!cR?ygF3Yi7NXij>#-qoa-$7E_J8T=n0Kte9e5e0q`oAxgq1vr) z$mo^TNjJrF-PUw2MJ};@qST#OYay&Q+hecEKxL$u z^?m6947} zRu<)$(u8=q1Ip1?oaY3JV9V8uE#5cx6ROfIBzJWN=(ibt=T74%bUA*e3zq)ZXb%H* zKbCe8T-jkR@S!=Mu^@QUd%=WP_GPO~2*}5JxkL!p!ooB6VfjAF)rJ@61q+k_7;TOH zs|_*0ak=3K#sdkrek);x10Qyc_!`zaS|vGl2_h2CrpaQNlc_aznP9*-!E3W^&!Ysj z5Isn;GosCqDA{L31-WjbNQ>2y+2gSO3jw@ZTVIkykAuxeXgS1#LGur`(;$iY7r<_2 zSVzJX+uNv}2MX8)K+9CqWo86~?co#dn?F(aYi0*2pM~D_8hYvP7=LK&qTZK}^7NE4 zLxiz7Nzv9N=N89bCa-x>MQP*VdXOG}VzvE1n927|8ue!9OYG3T&*?ajv#XCLAdPw~ zxTv9E-mYgmtND!CmO%6LK;^@W=VU(h$d3}wZM~2U3Y1^Hwsbd2`0L%TDqF$1Nl8Ly zw&x=)m5cUwtgPhzkVyOe5T8CIgV#=H8Z(kyANGD@yGDvuJg2(iv0_5KN)>%TVs6qS9|X5lVuj; zlqY!~O(`4^b6c;ze}BMXt6Ng~h%hb9GC7$%8_WKlR$#ICUXN=E^8Tlj5j%zt=zDi_ zZhue9GZ!UnN-}r1%U-L;s$I2aJSY2UE8pNA8Ctxng_%X=Y2rW0GLrDWniB~askmL> zGhz+zi2T|k0FNxMEgv>2k`&%Zlmq1C3I06CiWelMl>?Rp(bxr3Cb@|>-mNsMISFD@ zjrjR4w-fyC#~(seB^h{GRX+wflwncHBP}>0mjf^ z#AaM~?%42~@j+keqpjtz%!$Otl+NS|c_?L(EVsp$;^2=YthfZwIJ63uavz^n9-015I2TJ*G9r+D{ zsNEz>s8s^gn(Hd*hlA9=KFnd!d_!T=S6+Ox_j_oN_Slhn;@vZ0*?l%J-7QE!PVn|= z=jYUM<|Ra^VyzzDU5apWT=C;SRmI0=rGD^}%6!QtQOq##w3^0sF6HQk)Xq;E2;jkT zu+)!f3_SfA?E-%pV=3vgirX|wCJnf7O8lBoR4}RS#!Nl=oFC@ip2_;O2culZ$$pUe zA0>&C1Tt5DR8>37*;RtAYpOgdWp15k1`nNq0!)e1T|Y+mxhqy%@3tU$*H18CrSa+q z=pPV(gOev>RlASAnIlg)R2+MWHmuQYp=;80=;rb<3>%N>@O$gwWw&8-&j66O_~Dx* z-NybS$AO4o_x30r1LyNL{Fv}B<~c{JNP%A^3{5DBEc6MKqIRjYj{d`F3dR%%00jm>gRt>mOWPuw=Mar2=CYHo;>$z-uKnME;0dr+eeKO zdsL9WQ0>0T6)mCW{Gzt@L5d^{8;<&&-hpYRh)xdj|4FmusV+UyKgDop;7ef&n#Fsa zo#LEq^8&!q_oBRpkTRaj~t#$?Jw@ zEz#jFf3lW~E-NC3j-Yg)L!a@4AAOR>rfYfeq~yLN&%&SM(e?=MnZ|iq3rzE=(PN6E z;#t9eT?QjOu!=(f)+;uy8%dsyS5s971O8cP|LhR!>X1c&))XF7jLl92=`nk_?QF7J zT8yk75U2o50Y$e|vI&iTVPoqgPdgdDy}%IiR*u`KOnIJMhMd$^GG?MU=gVHa2*r&Y z-dDI)D)UcZGjR>HFVLq&u}Up+7F{8vdc8X=zyHl=FxF;e*jA8f z2HF>JEL>-==Bp?-vYACOXL~~j^JtFNzhb-9a9Pcc{36Lt^L)0DJ%tNtJd&`~^OPIc z=v2?V`fEr5ElC*D&djo~LV7ErbWqPd-Y;06*h8K86m``nqWkzu_jlbEdXCxVev~PI zp})@U;k&xK0K2d@0neJan1$M|bnD1PGO=QPp}VUsz`m?5=xc=~bJUDAWkKfqyVIXM z*Wl;4155;W!@>uFgn_DI4uhQg>-u8MXY9_HKbN5myzah1&DG4iuM6+|T4Nl3c8ewQ zh#_EY^Izbd<52yIiKLCRruATl71%X$+iyVOWA$ zY+Mi@jLK`1iD}aa+I!E=q7cFWy#PtEXg$pjBH+r|DO4^F<@e-HG>V(H4ovHFG^M!T z?yPQ#5fbL9hHFxjQ>ZWrZ2N(IdrTN-AWj!;mlxyM{&c&8pJfY)Q|S;j{uDMO*a}pz zP_9>`ogYNYCj(yH2-6TJan-O?Uniq}T4S+u$GtUu49DhA`RIx7fur}3F>dHk&({-! z%RYpY43#to&H|01YAM7PZ;w(Z{bwXX=6{bK1@7oL#p^8qE?xdc{^Q4OtKrc}G|!WI;@V{CrK!v(g! z8Z>wfclafy2h-yC$#C)Y{6gYhTI1eb z5t>F>lK1&`d}C!q};W{%}zZt?Ic>pw?g+4;lWbqlF>X0e?CN%RV!A z{iAgkUdRKVTvs&6&8M~;OJjm>OJ3*Z%Ov(jc;UzLpn8K{bPpq+J7X?hIt;7qaOoOF z`D8t+0ElU0q3=NCj zuT!7$Yx%3SW3itV{FNocEy~Dl`Cm59qDu&nQ->{qeWUhJVHoHMI)Jv(y+T4<%&#}t zfnMqJ`1@NAH8Cwu9etH2j&0x{KI)v@$6FzRNy6^F?X^bDMqNY^o6{@oRnERx$5SsEl=+Rk)-PQy^-T+@*hI5$Qm9fr_AxzL zojd9*8!45E89I$igQkNGgrgGK));mDoxUl(vj}X2_idE^5QSy(l6}K|wP!`C0V_rS zxnjc~61b|p`*B;m3skJ>`c1^;o=Kg=Fb&J{Yn_8jJ&q)5%edU>!SCn?VLNU#Gsd$+ z^5YufMRzB~EhV)-9POK(_=<{82S_LyS;qC#0NfiK^jZ3@jD*9l#i=yy_dUqahQ9vY zdUnI=DDso%u!H#nx0;WCw=lCh8s%HU|4(6G85T#hWs5Y>cmu%$bmIgM1b24=Nzf21 zSa1*S1cDPZXmEnNYj8qvcXxLJ^egVn+_`t=eedy?qNsD~oL&2@z4lrd)A7T{^a%Jj zeQ=vN!JBcyRxe!KpfAFptYr_SoKwA6} zI{%F&!fYweY)jhm+pFaRKF&@Tq7hJawX1c=ffyqs#7n3w}mG@g1MGR@L~o?iGrw?_Il9a(yJ`3 zSM9vYWyXapd+#U36#RBdU-SKVz2=!Me#6eVhRo*i-bc0HV~)9MyWW|N~H>+>~`Z`3l_Hktw6==vYlfJ}r>*pcP+bc%Jp| z2AF=qWwEWIR3aFgEI=+Xr+NZ+Z?`it>coC}=|1^f4sQYVvfa1yDe^l9&!aJarzIy_ zxByj-t?F?V%k=vVhBAdm+nJJ*geUDngF-%thO1)y2jzSR1IcY)>x1`<#n0CA69|hB zw`;#wd4Fv0W(8d%Z@meVzIUtgTmo?&B-(>tLycCV{{&TwrlIk-DHL=%MucCcn zmCPwdA^uRLuWDBa7@e4q6|b|`Qqmt|hBBpQ>&h_)D8;%!aqJjJ(2y+OkLzj6{i#1} z^37Vf9JiPLiY`x&55CpZd?>nenru@^+mkIAh-wA=dMclo8bTNzTghgoXwU7$W*H!@ zrg=)qYk`oBHNfg(ZT#6F-cBYHh{PmcL?$h;&jbT9Rj;+Fk|yx(qw1T!J$k2QOjExw zI<{kGcMjC5pV|#r8XKQ8M(k-0$jjSe=CWm@u5j9cYVJ1Xhb6mT*pEZL=GO5i-ff1% z#?Y82x7oozfD4P&SdP4VnJ!A@8Xl5_fgG%z1q1wSPT9$+!YCGu2pi!AnbTZNey_7- zGCw#1V7Fnk^;7w9N&_AmZzFqqh}X{^qhfaMyDv=Z;NgVFLR{XUpHzsRzqtxJ+q(=1 zV(J4*q;!+01O_0nDOQ!C>~>J3IuScdZv^-rLPwp z23c*o8v#n@EE4^3br98uK0q~(q0(77vDkp9bU105gNU*IW-YAhsW!p*HS)c2w-xw3 z(SmPrca#x0@I~XH@sk5n=K2P3{4?%SA%o2o~5!Gu2d;k<1!#0iPsx6N4BtS<-BNw+wQfwmJ9SMyB zpC%YHydO?};WnY@XHaE=gP$W6XWrecM+q8+9tCiGV(oKnY@lyP`ur!Ae{1;`trbDphp0W2A$=z*L&VFS9f^1BVinhik7now=W(j8 zl`v7a(sjYD4BJOzyL(mCh5|ZE_cU5W7!(xf3xWK{&^sc1ysqPw?I*?lfZ)gM=Ud2mMM|9*pv4E^}M))^dVuuk-9(sm(gwD zX0ocL?aMzuT*yBTKSia7{dmnvpka2QYWU)j>kl7|@h6`bO6Vf@C%>xtzcC1&dH_#+ zP5oT^{Sn~O7uoG7RSQ^Xnc<4u7`Qh@VnXaHhX#CwThks-qkQ=Cq8;#Iui23%h~*^i z1K8W{&q|RC-;`8$(y@9RzWTvN_30^1-cxoT{8fQ~;y;-5sq-S2)42C$h48UJhe?sl ztx0OP#dMEqv$Ia&*qK>PV{(i!O2a{3W7+egj zxY=jAUE$WpEoFnIL*L^mfZ}P(bE^RMFxy2S791&2TJjQYbUJsBnsyI5KX0W13h|F8 zKsZ*6pfd^y4mSVw$yUCK5l!^o6m@p|6Ov7cgb;1zA+ipnq1LU1DJG=kkI-ceWe3Ai zyyk~p3;vC0$w~^BnGRczogZH+fwX-R&8>FEDC|k()R7qLNq+tJmqXZWn(IQc*9?pK zu)uz?Rl{ZivnYludzIO+3)Nz=%5Nxvdy$-fm!5Ua+YO)y?P9lqwVbJ1%Jq5sRITwanjac(NS@xW`>8Ra3`aX;&4;HvZfbWDI|eel#a zUG+qlm=)&%I{$obBx`|Cc0IuC%>k_9>qe^Iq+F&qMq*k?dKzcBf{t6i4*<&P5B+VK zVtdW%ph(&pJiE$X{|CU%@cNxb;W$2Qgb@i+KbYi!KsDmuRa4To0f?W3vwB`F%M2{V zL{N#griNdJfA*Rze*8s;?Coq1I|DeS(?ke=J+Jy(uKxhS)cCsQr&XIC(9hc`%SZ56 zRJ8z8poU$e_ON!o(bJa9F&bW)tY139)!T(e3X`QrnuP1D78qpM(Bu zYyRgx-5~-=t+^=5J6Q-opP%|MT2T7M(8xAQAD$139y8Z&8@mI+udBJIw`*HXi;OHjP}Ttjd4ohRQ_ z0g9up4+<+XY$^h~IR`a9aKXQ)QDxzZv+fQcG|yav}b1OMmy2K%u|iP72+6Dds7_6zjS+qQXK0hSB`9vh>w(!Q5J3}hnT z4;%--?Z*_MfM2aGSH9EG&YkYL@X5)~XYc^bek+ZKB07`J(s$o{d!iS3atpAFAz%(* zvNwNl;VVmY3r&+Ax& zbGAwPdZuV^W2qibFzT+-E`sFa{V#3>Y)myixS6On&2B3EZ zO8NlXz}A#`fYp;fr5^wI`t0yLt!mv+CwcdsXU4AvYM;EFZ_I^F_9iMPH4|wa+iBOk ziSxm?0sM~%?-JOKaLz>2dpwf44Cc)htCQ|R3 zC!m-;fwuVFn+|78|I1JH{5M;`l0`lBqXv&r`<-;N{zB4~v&!(g(zaR&^`P`T5!Y=# z1-m+O+!pjpk6zr8VLn^?s5zR`VwjKo!Vo?ZORaLuvsWJ$=|6npv0v*t0K}aW3DA&v&C--_?@=%-nXS|+ z&#mlXfL_upkV$$D;|2~y@GkA-^szM^tWD=DC68Gma)*47BkNpCe3MXba)W_ur*_Ij zlkBqWh3kl3xtN8BUNo#YM~{aT0Y%^^kvd`!y_MHz#Hs`R`cij&P8t?MhiOR^@)Z6_0o1Ge+j@CedvG;opy-9yR4`lG}|==*A^xE>)j_=@1^yTJ`r6 zrB=NzecInxmT%pE*CR~xKpMzxs$&RdISsT}*~TgX{q47yVE#&+)k=wpX0E@ zuZnARZpX^3gi!)*IdcR%-Jiw5oROLJXn{GEubrY&$Zu@P_X#9$T(CHHw*05Y`e(HuUB2qG-2L>c?k z-_MY7pd|)|FO!

!Ke0_;zzU>?0xwuut8ctt`8z`{2>f0SvKr7}AmE&vHO6u4)K; zD+`GIo(O%L9h|5IcCx>)z=0h#&|7Me`@FL9P7H4aj|lPV7#*bSAvVzufNUYPP&t!cUIR)CwRId*ZK6CQ_IpnM zXe#%C1n*A2KUs46juo|yJ1LAz zzoUAg#0$Oy`@fuAmfl?>FJDt&o^*25aI!72yquWUv9CV&xuBPUVR7^nuepfW7WA;< zyA~vPQrzaLT?D_<*x?@cT?^p#y1O_4J|q{xgYa~3_gbeyL@ik-40U8gGH**j=H2MW z+AryKU3CgX0lo~Lu+7Em8DYVLkzWf&{;2i3@$>02tx{lae=&tM^~BbXm{^pq3-+`1 z_UrD>SY)A98Z#|d-=ixgX_S6+Q`y86*G%cyJPa4y+?=ixSK(wCy8@P9f}9bT?leSP z#=Vh4-bRm&yZ0hMTcOIF1V;6n&r?b?FYIzTjC-(-S-56Ga9y8*++rxg$LXqJ5#3tq zZuOWeB}jU-W3&h)wBF4S+N4W6hjlOtw^vD_jtm>a5J$Iv7Wm;q-PSQS5+{rE=n$kK zO7yXQIIle(TT14`G#rrVKe^&J+{YWEkRSy$pfDlYql>F;yhKF8?4XY4$U$4)i%y8H z?TXjYlWtwtz7^KW^RMVTn5`OD45QGz^ZA9%%R(-~t?pXj3So`w1~Q`~w~#Mu)c*DU z&*>Ik@W{!xF@lIm3Oi7yIUU^Q=a-1Ybt-#**2S9GY9UL0cM9V`j^RNB+0?kVBb@1n z@3>8w5>rpWIBZPX&Rk`v)l}4mVIa-~Lk>E*g)B=q7T91R!9|et+SaJWF#mEueunql z<(Q1=DwWsjl~75m)+p%&)m+x3-1|Kq3)$grooAQAKU@mj_bd;!=E(i8)2}o9l$NTR zTWycWzo;Cvv}H*$EDue}h_nQcs^45u=tdVM$NUwbDl?GC zv;jT2Y8^O{Yb6O>PFMjJO-@LOIGC5N_1H2w5yy=*@5EpR;7qhaMW^x#jw~ANn7?|C z-(ru$UGdAGa?{yBbPqz2SQ&c%$=9s=w0T06uV^;f-c0d8hMtIrF88Osf9n_rq=~ME z;!=40AY3{5I?4 zJ*ER;EF*|I(H7gCjrRSz9S;AWDnOPnt#U9GfB`6}d2J0-dxAEZI(--2dFDeM^ya{e z5HOeA5_QKO@migndB!SHft=j~0^9FrO!(ji859W|MNaUvGs{vO-^X@DeAIi`ae3$F zJ*z2fh<{l@&Md^X_&3ysR4LWgqtnHb!sc1F>91kV*s$vcvVQ4>INc~bVILl&cvno! zQw>bySxyd#g*fcP5U0$n9UT#hIQ~`WIbE7Kf>+0iX6`Gpxp<5{89L$ay06kiF!dsI zXL7HpaGRe*?C{Aq`pn&GthVt-7FPnxZm3+a7B^hF5?LT`-shxv%jwQQia$+O{u(L# zE=wyp7&AExSCeS^_Q`#MF#BTwhNnp$8|X$%DP3r`f2kJr&uLPn(>$z1{txBD&NfBIpS%e;8>2PKQY z^Dv(5^HqvlJJt@5!-RSbdVCgOdS+y2 z1Y1{#u@qqg%|YDP(W2#@-=Doi8w)m*TzLJcg(86zfM_q~j#e1J9A3`B?(W@G>W91A zzrIwu{VpuABS?#{MyLKJ>dacRF*Nb!&1zb|;%JR&=f?4<+cIRQ`V>2%c4tk2maU;M zzZ6nRc=z@5*PJEpoa?D?b=;oJkuV1U6hZ1=<8!yHUE^pJ=)$w3`!w|)tY<m8~B}ygTs%w#gM?0$*G@!W%1nm7MUxV4Fq;AUXrbGG$VLT~!HF#1%kUZq@ z4`69X2`YG_oi3X?8wJB1Uk$p6d5U`1b*F^cD;DT?v!v^h$%0Ih8mlQ zGCMb$`8)py#Wviyi$B3LQO_m$+jR5%2YM8WN*5u0dkg>+)CNHZpz3bZ6$i%jS_&L5 z-w`J`sq_{F6~a+GuaKl21dU8Xa%+iaV-A(0Zv%)+zqQ_`>4&N*HRQ=xf2%}g_EU1F|(^UzQf@{cgxUK^)ZvfuzAyNH8 z06ALVy+7+UVSF0er*mAJS0v``tWrMFeKEp)_IvKH@L4)>s=b<#ky#g!1zM(p592ab z?fs(nj#Px37KeAsnu93C*e?=otaxi5NZaHV9;JK`gazpwB1lhqXd<*c(>QXIg0X4Y z!X8(+DyOngks0?DJ|)6@r7^pnIY$KL*|a^lT)Ps!$urN)Pa*rDr{Rmt&F{2Rs5!wU zz@#OuhI>74D>|%}AM&2%{Rx^ZQk5M70_}#pl=xd+`GR8thNDg0Tn^-20WITOPryUf z6gb>D;&-E2DC9IQ9s9lgu22Liz(thP%4>?^L`Ec{HyLFS$bxAFjd6A@BGA&oCD|l? zi?@%Gnu#Sw(o9v0v*Dq+r&>mrE2Wi5tvLV=77};(Y@H4txW{2$DmjEm9EA693BK(z zF5!%eUakf-OFhrkJc%W67zav=#`X<>(5c$8Qw7*=z-pca2YUAuzkWkix0!-^f{2VX zf!;SQgZk$I1p*cT$K=a4!p(!`WQ-Y;(Ws$1b<^bCY(V%o1oa)SP9O|LRJl1n}os^V$V{r*}S0NYgbX5va$Zmic}vYj0GsY6OE z+_s~$gLqPWnf9VlfBGj46u(YDk$qO*byd->827!}vDj*T!wq*Y+4J`sy?iJ78$7sPL?p$&NBi}G+__=H8gVY)Ns6~h<$T{1ML`xH3BhlNPUx?D zVEx%)=so}Gh+2?u``08KcLnYVcbq_R1ppZr!xs|@^ik_$=|dc&_$a9+FlH8WMg0Sr zYO3P-7hi^HHdCPia9G)5xzV3x%w)XHGY})pjP1`=-s&c{C!_!S|1?eNDTCbNp9v)Wt zKtdg{#|Q$^(#38v(Seg=m>-V0m9)ih!(-}_ZRp`Uug8=`ppI_Kqa|wjYqz(;*U|KS z8?L&IR9*)am905TcvQ~ppgR3HhT;4JCe;Db;yzI+_GSJfm~B^D#t_7w@Q}&OF7#!N z66wsxTBqFRcg)NK)#^{HcI;KUavORJ)Iu1E+FmzmH#3q`lq1h0_I`Ul>p$Ms!g1fy9 zP#5lCyI7EQxn`o}RL01a{J7$V0Z$1N}mNpULAHm3N)#}MUv&iCkK#<_h7rwj_;($ILF zw1{P#r(uV6$(fOvxhs*EPa>h#zsbZ_;C~JGpD^KMOGjrywFnS%3L0T6sL3=BwBAe? zP;P|TLHD8j6?>R#W%0u=zV65KD5nHFNisxdl#}ENLA#*R#^r(6Y)h7BSm>lvThhDM zS7atbP2%4rpqz;n+dsZgV>c^W722$&nHbOq!uv6&ilQS(l*B))A4A1q4i?8JEOXMQ zDrbLs9H~p4sA$=dpZno9f_{f!GAjo*MwD)*p>Uzv^5GyQazw{R_w9hdef7lDaqE0O zJ*W_pVswh3mYg()BB`~kM0gSRyl;8+F^Q96kf)5dh`}*!7l#x>j*alHbniycHaGLm zV|)5%5-IUSnjI87`JhI`^ohxl?TvUlPqQ_$?XK<`nLslM`Y$IedkU{QFq(9vdA4;ip&^{yCT60^$jGB{Z*aM)o+|Ex00CsCYI{$ot4!AK+tJDLpI_TLH!;di!J> z_f*#rOl|@S1Gw(taUHENF*BgSwbcGBSmpVfspE!iO54=2@(i;}tkXtwDzjtI%$GP^ z(iAChH6|k2jP1w`v-84;+s#Ph9MPoUU6QfqRR_@6wmyD47a!a~c?{rVNl(?7BS(_9 zwo{e!MZ@TxHqooJ7y#&`Gi<^q^h|!Pm_o!R1NAS8>mYqn4nGjg3w!#f3 z6J!H$?5jE*cfsFy<|v}Y#d{3g67qQASRr^|%_GBru1sSUA+Wt1L?J1M(t)@7^R_D} z4%3LOv&w9Mv%a|MkAO)m<3rn<-W(X~evrmCBSQ$^K=NJ4=KfZ;Y+`kDh?VZUcx z3V6R^^#V+uDaM)ElfywfMh3{pICF-OsOWdiX3=rYDQ;)sm6o}gnC0mpL!4F%K8*x#wPm*yw0WKhXsuUk1fRI2qE$DCk5+P~6bq8e+me~- zM~+`WaF;Vj4pX<`WlpS7RIBwQ`CwO{eao7>$&u~2ANT`th%#QCY{eE_HepS)(c(-4b9VH^*F(hI{ zHSefOqPPkw27Lkh1T%+8oNX|Io*kz2{zi(j zySY>fYMvUfC2M8x)#?>jFEzhC#zjXWMPjByDEG~I+^Dc{CSyYDgXHnNffslf&dTUM z8#Dkf*btTLPJ-bojTHWJohMloolW2hcxv}BAo0eI%3ThX6+Ky)=&YfzSb4}_sL0#& zqEn8Li?-Yes{ZiBTWJvE0rKVw_EXd#fHdm6BD!Zomgq>pkit$y)?ayJthijbq|EsC zPkM-pqhcxH&mcv3K)6g|>oQAEzmM?cm*(?GYA<=)?~$#PvhHJ_f3Btm@9%RCZ#8v# zj2d&jB3UuWEd0*>Mq_e~`7X+db0z7IKNCj2P@&x`P2~u1KSC81(@NV0C}7 z1)m&xpFH0_B-V@=UsA7A^qL@*GWJA}CoVdTE2|itES>*}DtBFCwToz7+WTn1c}@^wI9(AuokVBqBa{zKEzFTjxH?GEVPxQfE;)pYpgp ziN?wiwKmJF8c~i9jAS;1ekvy_HnV?GTmS)6EXW3qkmeI83Ay=glz!3bU)CIUQ#vhS zup&d12tHE?s#K6Kx@f!S1mS;VP3UtA`vB(7OMXLnY4AO&Y3UqF`5Cp9e)m_j_dp1o zktReh&~ZaMMS;>pT$@mv*rOfiF(=1BD8U_Fe8^+s6MAJ{jB7;wJ^q zBgujZnUBG;oaazr5aX{MP|=FQu}SXdbqX1H&L!{NAFY|-*VFbDkg17kug?sHQ9J1q z)I3KP-RZ7j_Z8A^`0N(Nn^S=&JItt%F`xpW{l0wz1&IisQds}3=K1P+>K%FGkMxV+ zQhM9~wUI>6j4@mD5>Fl-jT&GvWC#RGwXkDkTMj*?({2Y&X8s5mFL}toj0z$J&&To1 zsCvUQ*RywLjH8_QgXfRyg;RvwDt!vd4ah|wCAVa)l(fYeqDEH{WD%$aqGrFg*2}cd zU2!_1whi&DTVdR(%*wyX`^c=d&bWt>-2kSd-^0fyVX!{b+Bifg0|(IFnov*gFCOlX(h!)JH z-cN{^cS%qo;WNLZy=&TSnb@tz4fAxOy7O~Ufu!x=gbcsEiMj@lxl}joiPa}MUT7?& zEWL4hK70jdVFmAE!MyqagVs=It~;JSt;aB)*@4jIccuBKb~qInhJQn}v9c2@k{*h> zYnnH+<>bhRzr^Jpov^k`#BHC71VwHN1(vUnh`8*_+nx}tV4HmkBl5yI%K{)A^6awa?e7`kf{mfOm&3_3W^nw!^_N8_6o;d?2*a- zs*MWd>eeo!(!?e}<>d{}pJF9k{tZY&OhqHWHRJuL55JrC`gvyfg(BezZ7M~}s!+)m z;Aw^JAU>h-$i#?!IADQdQj2{<#IU8DfaDC+p{C)tMgo^ko)!$C{tlmNau~zzPVe5< zDiuEUD}i)Kc{RWPdyEU{_D*qC89f9a;_imYEs%f9*NA23_^S( z=DzaFB~mZy9%2?lNWM$|P}4W6a7YLK+J_7hbE*$J0kj_I-{Qd8K55$U!Q*9hgn+FEot__j#h<#(oV2CiEA0+ z;~1Sv(`=s|BW@1Kv{9o5miw9nEUe8Ms5!C1J^R|)bkYmzuq5oMjww$?KLs;IH#VzA zn>90cVV6eNG;1~o82D3$UVr~Pf_JZ)9(I8;O#%rKN1PL%@TvXe|L0>KVrNt53feGu z-`?quuJZIqZI%kOtlZ0~c*$I_2MDk2@i7mS9x(_9|Z zs8J;5Ke=Z*TXxJYp!TPpYt%yq-5xT_88^`V_O&Z@=@v{al4JCfvh_FJf^`i5mQo6oQlR5r55haG5iu*9w20(DI5su2iYTh}>I zoGerFeZ36bEr<%Y69Kx1T1Qv(?YjwoOSm#qSxYlj%!rRNYF}q@-n{8S3sE5>s%7~k z+rVkWYkDwa1&|$_7A4lhg6pR-uWEx%Rpk_PmOdNK32=MVXd)#=p3MVbHI@|&8UbCl?^E!Z-d?9UU$)*aR z==8whjNB|~dR=b(v@0G05ZtjVfl!jfY5d+yxk8%@^2*D?N8D5V2E-pTOff7?TrQh9 z*1?89Cd1H5Dz5sa#&Yx_2QLc~-%o`Tgs__%pt>E0W2a>{=9V|EKe0=1BQUYycQIDc zqla{}ii3#6NH+$?wCHf`NItn3gvv>j<7aze%_cLHl^@XAdb&cB%XJnF0Kxv;nqwvv|IpnDNb}Ob56j8ziH4=_1gx(Sia1J~}@-Och{c{_+2O{LN&Ha12?;oU2I92VIQsHlXG)I4)K`t_>?p>+ zi=t^#gMxt>+GfWN*(7o+s5WyKp@mF^sS01l)k#BdbeJaEWU)L3Y7>fLP1AKRmWcE$ z>5o*F-$Xrry`~@QX_{qH$I|N+Z2vn`a6f;sEjMm|xsWn!JX#fE8`!PMdR#N6D16G} zwmaYHxF@f#=H7U!bHPuhtV?6keVrtrrg(GJj%H9C50{y2)P%EFmTr0U`J%9GLHEuN zGv{)lv_WE25IYm}P(zGwJ{dJCxHT>%R1Qb@j5>Pve ze;AeNFi)~EFoZfMyGzqFCZLE>Rc7lFfJl^c9L0@^JH7I>=O4S}-VcOFc6WsmQ~?gP zLxR*y7K5@peI8la#kV2qvI*rwQ6;4#1__haNcA{PWQcpLF>T)KoRM0<}QQCDLNH*uB0#tN{jb#%e8=?*U+o!dblGIdjJ;rLACWk z*YH?q8J#S|A%la>@9BLu8nOXZRiJ%q-Ji^Qw&Lf^E7sk(VLqZ!pU)PKQa+pHDfjd_mDz!nYwP z_GYz(MK>_v$<2px%j!mKPaC@to^D3L+VglkAs8WR$ub(}3(dL_7p;DF;dm&YM>337GABW7 zBpgFzsVAg^E0RUIJ1wk@Q+wOBG_CFznme7JyD?}7j z#?Y#>tGq8SJj|jX%d3yMIAcz>-@&tn)k3lzf92UWGDvO544b&#uk1P$4F38<9Ay5i z62*eNlxWOy)GoY0O^NaS@;s*$dV~ZSx>9a{T-V(HX6i1u1>pH;AhR({N|iPN`= zW1v9k(2>l@ zps|6kbgNE~8J7iPBbpLNeaCkpXRp`KG^POt7|DC^ii2OrD^LD&idgPQ1nVfBgnDXw z^4@o@)euCH3oE6PUY&1|EgAFn5Evf4mld~40D_pb7{bd<+|3R!L<6McV>|%yU~#IL zFrlR!{Q6Fl!X&#JewaNP-dJZmr}}-%MkvqluGv}SWIg~+k!K=oT+}QvBj6njqX2_- z$^&vteq~I(nBwC)3{Sk4-Xu1$F!}P@=0|gOs(;9JPkDS|MisIBe8i-r#U2fk6UZxi zK%~d*Wnw`g9P7hLDF^E%fLWe*kML3P+N3r`%^F>7X{$yQ{`68Z&9If#N{ftCV_0a_c^IDd5*2$ddb2iI#H z&2*M&QX*3ubg{%6wvci?DY-U~)|v8ow}1+gNpqa$mTvtKmEsoaq=sY#A%noY7b72< zozmvj!`Slj#CRkGWEKuchYa3=V`D8$AuV+aFMnM_612Dt>7@Sv>diBN=C zSn+9mS?M0|k62fj!p+HVLvcm9$%3gKocdhWwXLqeZEaef^L5FQ-f*bnR16jHdusw? z(L6`2bGWiON3M%iZpeC$b%TXAr$&Dso+QB-^;UhsqS>DEZ+Y#drAt~1u!c22B*g~5 z<~&Ifgy>#Q_bGZsp3Q!LQ@u+h=m7UZL;|5#C=J-@P6>a?bckHx^(i zPP-MgP8kva1S5x*u+GM^fzzsXX=w7_Znz5T~=WqRR8rIXpPgKMb-Tt7I=0DNh zNsNTtrhRYA2Cbkra?&X!8sp~?8c3o=XW@GmX2q>Ea@0WysCUP{hn(cpg-L& z{_>&7*)5)}s@yMa`V;yFHFiqJvZS~dHJ!xQFbZCHmievaybxaI?k6+cwSwCiP58Ou zkJ&GQjAZPa7eH8V?GwGKh%#YFwe!!1q0aZIzFOk}zrXr)Fsb_zg8}r<&VYXqHQQeU zyn>_GWr%vIJ|S&TOqeqsxT{jS6c5Ea{ot@O1wL;d9d%$#lkU)*Yg}wsA=8cV>hVi7 zOB$ekN|i$&$V49ag1if@wE_Ajz{+cLQd0UbcvVPVa4+% zWOY`ZCegux1XC!)=-#&?%J>f~4(2)|8JXjTN^WHeqpnV4 z1bgB0tWA6XV>yGMj-w{c3_F|*!lUSUzx*dQ7uVCUR$Kf>g31@i!EcuMozN04ezW*~ z+$3?u`u*}|Uq53Q=xTQJ86HK!n&OZ&WFlAUe1#F95M91QS9~?9d%iVP(~+U+h2n5Q zEP$rT#avo$G-V7rEl5{ueJLD9Xk-{xO55^7lJ5`hRiKDiSGn$dKft?xpGAPB zM)d?c8s|pq`65uj4M%U;wW9=L3@nUoMQvj-qp@=LwayZvSe2r2KQak#3DMG#p7qwh zH3-|U*k0me%VDHGJdJk0+F*|1JHMa1oFSl%gt@KpwWNPo7f@Fz%ViY~b~+V`D?Qh@ zdZgOU<*@fs3%?@-A`lspin)$2@4y&`T$~ej!{F3(y(p3Bw`?RU}5SEOWp0<1zSb|oaF@mWwrn9s# z{fd??Wf$`?@xK_o|IttKk5wb)_;jMh;a!}h1A~6VsH!A8m5cJ#X1v5h>{#8-g%7|P zw!KT9Sv>Oje88Bplx_TkZWvO;XmuB^aN5;kF~ePYZ=ag)5qS=Z)?at8obAtYR8)Fj z7RRPX42($x^f}S<^+2EU(mHSW$YnBWUUqrUCfVJK#a>R;l_b9CnO;sdy&1cTXyQ%> z+Yf7VZ9@vYw58o{2ca~zP}{WB;N}|E;tS~oRHHe}e?0HMGVbq!fqt0Q)%>QWXMRxP zud!%Mt0@oK^o(mdm35*IF5Wb$$KUorJC_;Dr&~2+yGDXXun4(>alm$J4SvyGUyNT+`qN?x#`C0Q6mVfn1=V2PUQm&Y`YNQ=CF z2W!9!ZgVqDbxc)AMMKwBp<1mP7F|XZRK*B))W6HEE3vKY zS(OWLQztwm@pLlXy+Q}Cw`1~0l)=XH7-mFvvP@Rrb4T9ozKWETy*bUoyuZ5~Ar3pN zs8#N~!M^*C&Z&Q!KEK+h$FUxHvF7(x3_?~>#!JN{9!uH#5D{CyK-^{7wL0w}WpkGi z3CspbDi{>H*T9U|^gP?rlbPZ#+y%RT|Hpj;T4l!D>G-IM;t0P?^(4NiztzAy|6g58 zf1du6qt?EBz;nO26}c#&IVD~g_%z(%Av#4|D2o#LQnG^n=O)~NAnCklloZ?N{9iZx z_i>m)AffYEpBiQVN3+zwT?No(U{4%&u(`0z|K~a^peC9ku6+G}*((3K^aKLMOz;y4 z%LL=V|Lc~1N8-NT?c^6|R+ypZ7drR(Ex@_*pTiJ42Y>$i!oroXn_I}oeyQ1QcBB7W z)ohiS*{@*i^LJ%6XU>##Y5%_1{W`+hJA*K*+|IX69Q89&hLX7$Qh2O&csy@TtO|1z z1`^&o$SsG36Z>Ct>4bv_UO>Sl`iM!y)|c*mXR&P@mfjkOa#(0uSohHA(Hn{l(juJn%OR%cKIgAoP$lu? z*Nps^i$~l*=B}K+I5B?#c>o4-VRxs7#rV8Dr8PE#>P0e(GV&Nj8f+wD#RaItc#0Ft zKcDhHCc1q^$~O{}7Sb@q{u!W3c&cU??B*t3;dF=K(Y(+J!BUZmWB!++Rih)K4zqrm z`*2c0GWOE^&jG{S`@*(HrkhvxbTwy*8JCcVTc*Vi99G008nH8qJn2Oj{=H;8V6Ak}5 zK~H}gK!A9MVC6&&b!sUysJcihB;Gg0`jg76`7i%_hXpCRezf%~;)Nj!t<7wj1TAhI z2Bsp-vAwd#zYOdjvk^>$q!l<)2rimWYG9y=`tc4oPWPYJWO9LTpp7wgkb3FEl5V*L z&&sUPgtk)0e_rsP8xU0#X_V?DGDv&7=XPbd!GZC=hw1kR`dVskE8`x{wN!%jCI$Z) Q0`Ma*ts+$_Vc`FN0D~LqHUIzs literal 0 HcmV?d00001 diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/README.md b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/README.md new file mode 100644 index 00000000..fd325bbe --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/README.md @@ -0,0 +1,51 @@ +Knowledge-driven Dialogue +============================= +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +This is a paddlepaddle implementation of retrieval-based model for knowledge-driven dialogue + +## Requirements + +* cuda=9.0 +* cudnn=7.0 +* python=2.7 +* numpy +* paddlepaddle>=1.3 + +## Quickstart + +### Step 1: Preprocess the data + +Put the data of [DuConv](https://ai.baidu.com/broad/subordinate?dataset=duconv) under the data folder and rename them train/dev/test.txt: + +``` +./data/resource/train.txt +./data/resource/dev.txt +./data/resource/test.txt +``` + +### Step 2: Train the model + +Train model with the following commands. + +```bash +sh run_train.sh model_name +``` + +3 models were supported: +- match: match, input is history and response +- match_kn: match_kn, input is history, response, chat_path, knowledge +- match_kn_gene: match_kn, input is history, response, chat_path, knowledge and generalizes target_a/target_b of goal for all inputs, replaces them with slot mark + +### Step 3: Test the Model + +Test model with the following commands. + +```bash +sh run_test.sh model_name +``` + +## Note !!! + +* The script run_train.sh/run_test.sh shows all the processes including data processing and model training/testing. Be sure to read it carefully and follow it. +* The files in ./data and ./model is just empty file to show the structure of the document. diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/args.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/args.py new file mode 100644 index 00000000..4d114012 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/args.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: args.py +""" + +from __future__ import print_function + +import six +import argparse + + +# define argument parser & add common arguments +def base_parser(): + parser = argparse.ArgumentParser(description="Arguments for running classifier.") + parser.add_argument( + '--epoch', + type=int, + default=100, + help='Number of epoches for training. (default: %(default)d)') + parser.add_argument( + '--task_name', + type=str, + default='match', + help='task name for training') + parser.add_argument( + '--max_seq_len', + type=int, + default=512, + help='Number of word of the longest seqence. (default: %(default)d)') + parser.add_argument( + '--batch_size', + type=int, + default=8096, + help='Total token number in batch for training. (default: %(default)d)') + parser.add_argument( + '--voc_size', + type=int, + default=14373, + help='Total token number in batch for training. (default: %(default)d)') + parser.add_argument( + '--init_checkpoint', + type=str, + default=None, + help='init checkpoint to resume training from. (default: %(default)s)') + parser.add_argument( + '--save_inference_model_path', + type=str, + default="inference_model", + help='save inference model. (default: %(default)s)') + parser.add_argument( + '--output', + type=str, + default="./output/pred.txt", + help='init checkpoint to resume training from. (default: %(default)s)') + parser.add_argument( + '--learning_rate', + type=float, + default=1e-2, + help='Learning rate used to train with warmup. (default: %(default)f)') + parser.add_argument( + '--weight_decay', + type=float, + default=0.01, + help='Weight decay rate for L2 regularizer. (default: %(default)f)') + parser.add_argument( + '--checkpoints', + type=str, + default="checkpoints", + help='Path to save checkpoints. (default: %(default)s)') + parser.add_argument( + '--vocab_path', + type=str, + default=None, + help='Vocabulary path. (default: %(default)s)') + parser.add_argument( + '--data_dir', + type=str, + default="./real_data", + help='Path of training data. (default: %(default)s)') + parser.add_argument( + '--skip_steps', + type=int, + default=10, + help='The steps interval to print loss. (default: %(default)d)') + parser.add_argument( + '--save_steps', + type=int, + default=10000, + help='The steps interval to save checkpoints. (default: %(default)d)') + parser.add_argument( + '--validation_steps', + type=int, + default=1000, + help='The steps interval to evaluate model performance on validation ' + 'set. (default: %(default)d)') + parser.add_argument( + '--use_cuda', action='store_true', help='If set, use GPU for training.') + parser.add_argument( + '--use_fast_executor', + action='store_true', + help='If set, use fast parallel executor (in experiment).') + parser.add_argument( + '--do_lower_case', + type=bool, + default=True, + choices=[True, False], + help="Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") + parser.add_argument( + '--warmup_proportion', + type=float, + default=0.1, + help='proportion warmup. (default: %(default)f)') + args = parser.parse_args() + return args + +def print_arguments(args): + print('----------- Configuration Arguments -----------') + for arg, value in sorted(six.iteritems(vars(args))): + print('%s: %s' % (arg, value)) + print('------------------------------------------------') + diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/dev.txt b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/dev.txt new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/resource/dev.txt b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/resource/dev.txt new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/resource/test.txt b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/resource/test.txt new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/resource/train.txt b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/resource/train.txt new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/test.txt b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/test.txt new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/train.txt b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/data/train.txt new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/dict/char.dict b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/dict/char.dict new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/dict/gene.dict b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/dict/gene.dict new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/interact.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/interact.py new file mode 100755 index 00000000..bdbf4a19 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/interact.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: interact.py +""" + +import paddle.fluid as fluid +import paddle.fluid.framework as framework +from source.inputters.data_provider import load_dict +from source.inputters.data_provider import MatchProcessor +from source.inputters.data_provider import preprocessing_for_one_line + +import numpy as np + +load_dict("./dict/gene.dict") + +def load_model(): + """ + load model function + """ + main_program = fluid.default_main_program() + #place = fluid.CPUPlace() + place = fluid.CUDAPlace(0) + exe = fluid.Executor(place) + exe.run(framework.default_startup_program()) + + path = "./models/inference_model" + [inference_program, feed_dict, fetch_targets] = \ + fluid.io.load_inference_model(dirname=path, executor=exe) + model_handle = [exe, inference_program, feed_dict, fetch_targets, place] + return model_handle + + +def predict(model_handle, text, task_name): + """ + predict score function + """ + exe = model_handle[0] + inference_program = model_handle[1] + feed_dict = model_handle[2] + fetch_targets = model_handle[3] + place = model_handle[4] + + data = preprocessing_for_one_line(text, MatchProcessor.get_labels(), \ + task_name, max_seq_len=256) + context_ids = [elem[0] for elem in data] + context_pos_ids = [elem[1] for elem in data] + context_segment_ids = [elem[2] for elem in data] + context_attn_mask = [elem[3] for elem in data] + labels_ids = [[1]] + if 'kn' in task_name: + kn_ids = [elem[4] for elem in data] + kn_ids = fluid.create_lod_tensor(kn_ids, [[len(kn_ids[0])]], place) + context_next_sent_index = [elem[5] for elem in data] + results = exe.run(inference_program, + feed={feed_dict[0]: np.array(context_ids), + feed_dict[1]: np.array(context_pos_ids), + feed_dict[2]: np.array(context_segment_ids), + feed_dict[3]: np.array(context_attn_mask), + feed_dict[4]: kn_ids, + feed_dict[5]: np.array(labels_ids), + feed_dict[6]: np.array(context_next_sent_index)}, + fetch_list=fetch_targets) + else: + context_next_sent_index = [elem[4] for elem in data] + results = exe.run(inference_program, + feed={feed_dict[0]: np.array(context_ids), + feed_dict[1]: np.array(context_pos_ids), + feed_dict[2]: np.array(context_segment_ids), + feed_dict[3]: np.array(context_attn_mask), + feed_dict[4]: np.array(labels_ids), + feed_dict[5]: np.array(context_next_sent_index)}, + fetch_list=fetch_targets) + score = results[0][0][1] + return score + diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/models/best.model b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/models/best.model new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/output/predict.txt b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/output/predict.txt new file mode 100644 index 00000000..e69de29b diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/predict.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/predict.py new file mode 100644 index 00000000..24c26493 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/predict.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: predict.py +Load checkpoint of running classifier to do prediction and save inference model. +""" + +import os +import time +import numpy as np +import paddle.fluid as fluid +import paddle.fluid.framework as framework + +import source.inputters.data_provider as reader + +import multiprocessing +from train import create_model +from args import base_parser +from args import print_arguments +from source.utils.utils import init_pretraining_params + + +def main(args): + + task_name = args.task_name.lower() + processor = reader.MatchProcessor(data_dir=args.data_dir, + task_name=task_name, + vocab_path=args.vocab_path, + max_seq_len=args.max_seq_len, + do_lower_case=args.do_lower_case) + + num_labels = len(processor.get_labels()) + infer_data_generator = processor.data_generator( + batch_size=args.batch_size, + phase='test', + epoch=1, + shuffle=False) + num_test_examples = processor.get_num_examples(phase='test') + main_program = fluid.default_main_program() + + feed_order, loss, probs, accuracy, num_seqs = create_model( + args, + num_labels=num_labels, + is_prediction=True) + + if args.use_cuda: + place = fluid.CUDAPlace(0) + dev_count = fluid.core.get_cuda_device_count() + else: + place = fluid.CPUPlace() + dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + + exe = fluid.Executor(place) + exe.run(framework.default_startup_program()) + + if args.init_checkpoint: + init_pretraining_params(exe, args.init_checkpoint, main_program) + + feed_list = [ + main_program.global_block().var(var_name) for var_name in feed_order + ] + feeder = fluid.DataFeeder(feed_list, place) + + out_scores = open(args.output, 'w') + for batch_id, data in enumerate(infer_data_generator()): + results = exe.run( + fetch_list=[probs], + feed=feeder.feed(data), + return_numpy=True) + for elem in results[0]: + out_scores.write(str(elem[1]) + '\n') + + out_scores.close() + if args.save_inference_model_path: + model_path = args.save_inference_model_path + fluid.io.save_inference_model( + model_path, + feed_order, probs, + exe, + main_program=main_program) + + +if __name__ == '__main__': + args = base_parser() + print_arguments(args) + main(args) diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/run_predict.sh b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/run_predict.sh new file mode 100644 index 00000000..3c88b9b1 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/run_predict.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# set gpu id to use +export CUDA_VISIBLE_DEVICES=1 + +# task_name can select from ["match", "match_kn", "match_kn_gene"] +# match task: do not use knowledge info (goal and knowledge) for retrieval model +# match_kn task: use knowledge info (goal and knowledge) for retrieval model +# match_kn_gene task: 1) use knowledge info (goal and knowledge) for retrieval model; +# 2) generalizes target_a/target_b of goal, replaces them with slot mark +# more information about generalization in match_kn_gene, +# you can refer to ./tools/convert_conversation_corpus_to_model_text.py +TASK_NAME=$1 + +if [ "$TASK_NAME" = "match" ] +then + DICT_NAME="./dict/char.dict" + USE_KNOWLEDGE=0 + TOPIC_GENERALIZATION=0 +elif [ "$TASK_NAME" = "match_kn" ] +then + DICT_NAME="./dict/char.dict" + USE_KNOWLEDGE=1 + TOPIC_GENERALIZATION=0 +elif [ "$TASK_NAME" = "match_kn_gene" ] +then + DICT_NAME="./dict/gene.dict" + USE_KNOWLEDGE=1 + TOPIC_GENERALIZATION=1 +else + echo "task name error, should be match|match_kn|match_kn_gene" +fi + +# in predict stage, FOR_PREDICT=1 +FOR_PREDICT=1 + +# put all data set that used and generated for testing under this folder: INPUT_PATH +# for more details, please refer to the following data processing instructions +INPUT_PATH="./data" + +# put the model files needed for testing under this folder: OUTPUT_PATH +OUTPUT_PATH="./models" + +# set python path according to your actual environment +PYTHON_PATH="python" + +# in test stage, you can eval dev.txt or test.txt +# the "dev.txt" and "test.txt" are the original data of DuConv and +# need to be placed in this folder: INPUT_PATH/resource/ +# the following preprocessing will generate the actual data needed for model testing +# after testing, you can run eval.py to get the final eval score if the original data have answer +# DATA_TYPE = "dev" or "test" +DATA_TYPE="dev" + +# candidate set, construct in train stage +candidate_set_file=${INPUT_PATH}/candidate_set.txt + +# ensure that each file is in the correct path +# 1. put the data of DuConv under this folder: INPUT_PATH/resource/ +# - the data provided consists of three parts: train.txt dev.txt test.txt +# - the train.txt and dev.txt are session data, the test.txt is sample data +# - in test stage, we just use the dev.txt or test.txt +# 2. the sample data extracted from session data is in this folder: INPUT_PATH/resource/ +# 3. the candidate data constructed from sample data is in this folder: INPUT_PATH/resource/ +# 4. the text file required by the model is in this folder: INPUT_PATH +corpus_file=${INPUT_PATH}/resource/${DATA_TYPE}.txt +sample_file=${INPUT_PATH}/resource/sample.${DATA_TYPE}.txt +candidate_file=${INPUT_PATH}/resource/candidate.${DATA_TYPE}.txt +text_file=${INPUT_PATH}/test.txt +score_file=./output/score.txt +predict_file=./output/predict.txt + +# step 1: if eval dev.txt, firstly have to convert session data to sample data +# if eval test.txt, we can use original test.txt of DuConv directly. +if [ "${DATA_TYPE}"x = "test"x ]; then + sample_file=${corpus_file} +else + ${PYTHON_PATH} ./tools/convert_session_to_sample.py ${corpus_file} ${sample_file} +fi + +# step 2: construct candidate for sample data +${PYTHON_PATH} ./tools/construct_candidate.py ${sample_file} ${candidate_set_file} ${candidate_file} 10 + +# step 3: convert sample data with candidates to text data required by the model +${PYTHON_PATH} ./tools/convert_conversation_corpus_to_model_text.py ${candidate_file} ${text_file} ${USE_KNOWLEDGE} ${TOPIC_GENERALIZATION} ${FOR_PREDICT} + +# inference_model can used for interact.py +inference_model="./models/inference_model" + +# step 4: predict score by model +$PYTHON_PATH -u predict.py --task_name ${TASK_NAME} \ + --use_cuda \ + --batch_size 10 \ + --init_checkpoint ${OUTPUT_PATH}/50 \ + --data_dir ${INPUT_PATH} \ + --vocab_path ${DICT_NAME} \ + --save_inference_model_path ${inference_model} \ + --max_seq_len 128 \ + --output ${score_file} + +# step 5: extract predict utterance by candidate_file and score_file +# if the original file has answers, the predict_file format is "predict \t gold \n predict \t gold \n ......" +# if the original file not has answers, the predict_file format is "predict \n predict \n predict \n predict \n ......" +${PYTHON_PATH} ./tools/extract_predict_utterance.py ${candidate_file} ${score_file} ${predict_file} + +# step 6: if the original file has answers, you can run the following command to get result +# if the original file not has answers, you can upload the ./output/test.result.final +# to the website(https://ai.baidu.com/broad/submission?dataset=duconv) to get the official automatic evaluation +${PYTHON_PATH} ./tools/eval.py ${predict_file} diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/run_train.sh b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/run_train.sh new file mode 100755 index 00000000..821be62c --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/run_train.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# set gpu id to use +export CUDA_VISIBLE_DEVICES=0 + +# task_name can select from ["match", "match_kn", "match_kn_gene"] +# match task: do not use knowledge info (goal and knowledge) for retrieval model +# match_kn task: use knowledge info (goal and knowledge) for retrieval model +# match_kn_gene task: 1) use knowledge info (goal and knowledge) for retrieval model; +# 2) generalizes target_a/target_b of goal, replaces them with slot mark +# more information about generalization in match_kn_gene, +# you can refer to ./tools/convert_conversation_corpus_to_model_text.py +TASK_NAME=$1 + +if [ "$TASK_NAME" = "match" ] +then + DICT_NAME="./dict/char.dict" + USE_KNOWLEDGE=0 + TOPIC_GENERALIZATION=0 +elif [ "$TASK_NAME" = "match_kn" ] +then + DICT_NAME="./dict/char.dict" + USE_KNOWLEDGE=1 + TOPIC_GENERALIZATION=0 +elif [ "$TASK_NAME" = "match_kn_gene" ] +then + DICT_NAME="./dict/gene.dict" + USE_KNOWLEDGE=1 + TOPIC_GENERALIZATION=1 +else + echo "task name error, should be match|match_kn|match_kn_gene" +fi + +# in train stage, FOR_PREDICT=0 +FOR_PREDICT=0 + +# put all data set that used and generated for training under this folder: INPUT_PATH +# for more details, please refer to the following data processing instructions +INPUT_PATH="./data" + +# put the model file that saved in each stage under this folder: OUTPUT_PATH +OUTPUT_PATH="./models" + +# set python path according to your actual environment +PYTHON_PATH="python" + +# in train stage, use "train.txt" to train model, and use "dev.txt" to eval model +# the "train.txt" and "dev.txt" are the original data of DuConv and +# need to be placed in this folder: INPUT_PATH/resource/ +# the following preprocessing will generate the actual data needed for model training +# DATA_TYPE = "train" or "dev" +DATA_TYPE=("train" "dev") + +# candidate set +candidate_set_file=${INPUT_PATH}/candidate_set.txt + +# data preprocessing +for ((i=0; i<${#DATA_TYPE[*]}; i++)) +do + # ensure that each file is in the correct path + # 1. put the data of DuConv under this folder: INPUT_PATH/resource/ + # - the data provided consists of three parts: train.txt dev.txt test.txt + # - the train.txt and dev.txt are session data, the test.txt is sample data + # - in train stage, we just use the train.txt and dev.txt + # 2. the sample data extracted from session data is in this folder: INPUT_PATH/resource/ + # 3. the candidate data constructed from sample data is in this folder: INPUT_PATH/resource/ + # 4. the text file required by the model is in this folder: INPUT_PATH + corpus_file=${INPUT_PATH}/resource/${DATA_TYPE[$i]}.txt + sample_file=${INPUT_PATH}/resource/sample.${DATA_TYPE[$i]}.txt + candidate_file=${INPUT_PATH}/resource/candidate.${DATA_TYPE[$i]}.txt + text_file=${INPUT_PATH}/${DATA_TYPE[$i]}.txt + + # step 1: build candidate set from session data for negative training cases and predicting candidates + if [ "${DATA_TYPE[$i]}"x = "train"x ]; then + ${PYTHON_PATH} ./tools/build_candidate_set_from_corpus.py ${corpus_file} ${candidate_set_file} + fi + + # step 2: firstly have to convert session data to sample data + ${PYTHON_PATH} ./tools/convert_session_to_sample.py ${corpus_file} ${sample_file} + + # step 3: construct candidate for sample data + ${PYTHON_PATH} ./tools/construct_candidate.py ${sample_file} ${candidate_set_file} ${candidate_file} 9 + + # step 4: convert sample data with candidates to text data required by the model + ${PYTHON_PATH} ./tools/convert_conversation_corpus_to_model_text.py ${candidate_file} ${text_file} ${USE_KNOWLEDGE} ${TOPIC_GENERALIZATION} ${FOR_PREDICT} + + # step 5: build dict from the training data, here we build character dict for model + if [ "${DATA_TYPE[$i]}"x = "train"x ]; then + ${PYTHON_PATH} ./tools/build_dict.py ${text_file} ${DICT_NAME} + fi + +done + +# step 5: train model, you can find the model file in OUTPUT_PATH after training +$PYTHON_PATH -u train.py --task_name ${TASK_NAME} \ + --use_cuda \ + --batch_size 128 \ + --data_dir ${INPUT_PATH} \ + --vocab_path ${DICT_NAME} \ + --checkpoints ${OUTPUT_PATH} \ + --save_steps 1000 \ + --weight_decay 0.01 \ + --warmup_proportion 0.1 \ + --validation_steps 1000000 \ + --skip_steps 100 \ + --learning_rate 0.1 \ + --epoch 30 \ + --max_seq_len 256 + diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/__init__.py new file mode 100644 index 00000000..beea2958 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: __init__.py +""" \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/encoders/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/encoders/__init__.py new file mode 100644 index 00000000..40928411 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/encoders/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: __init__.py +""" diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/encoders/transformer.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/encoders/transformer.py new file mode 100755 index 00000000..02d2aa83 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/encoders/transformer.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: transformer.py +""" + +from functools import partial +import numpy as np + +import paddle.fluid as fluid +import paddle.fluid.layers as layers + + +def multi_head_attention(queries, + keys, + values, + attn_bias, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0., + cache=None, + name='multi_head_att'): + """ + Multi-Head Attention. Note that attn_bias is added to the logit before + computing softmax activiation to mask certain selected positions so that + they will not considered in attention weights. + """ + keys = queries if keys is None else keys + values = keys if values is None else values + + if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3): + raise ValueError( + "Inputs: quries, keys and values should all be 3-D tensors.") + + def __compute_qkv(queries, keys, values, n_head, d_key, d_value): + """ + Add linear projection to queries, keys, and values. + """ + q = layers.fc(input=queries, + size=d_key * n_head, + num_flatten_dims=2, + param_attr=name + '_query_fc.w_0', + bias_attr=name + '_query_fc.b_0') + k = layers.fc(input=keys, + size=d_key * n_head, + num_flatten_dims=2, + param_attr=name + '_key_fc.w_0', + bias_attr=name + '_key_fc.b_0') + v = layers.fc(input=values, + size=d_value * n_head, + num_flatten_dims=2, + param_attr=name + '_value_fc.w_0', + bias_attr=name + '_value_fc.b_0') + return q, k, v + + def __split_heads(x, n_head): + """ + Reshape the last dimension of inpunt tensor x so that it becomes two + dimensions and then transpose. Specifically, input a tensor with shape + [bs, max_sequence_length, n_head * hidden_dim] then output a tensor + with shape [bs, n_head, max_sequence_length, hidden_dim]. + """ + hidden_size = x.shape[-1] + # The value 0 in shape attr means copying the corresponding dimension + # size of the input as the output dimension size. + reshaped = layers.reshape( + x=x, shape=[0, 0, n_head, hidden_size // n_head], inplace=True) + + # permuate the dimensions into: + # [batch_size, n_head, max_sequence_len, hidden_size_per_head] + return layers.transpose(x=reshaped, perm=[0, 2, 1, 3]) + + def __combine_heads(x): + """ + Transpose and then reshape the last two dimensions of inpunt tensor x + so that it becomes one dimension, which is reverse to __split_heads. + """ + if len(x.shape) == 3: return x + if len(x.shape) != 4: + raise ValueError("Input(x) should be a 4-D Tensor.") + + trans_x = layers.transpose(x, perm=[0, 2, 1, 3]) + # The value 0 in shape attr means copying the corresponding dimension + # size of the input as the output dimension size. + return layers.reshape( + x=trans_x, + shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], + inplace=True) + + def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate): + """ + Scaled Dot-Product Attention + """ + scaled_q = layers.scale(x=q, scale=d_key ** -0.5) + product = layers.matmul(x=scaled_q, y=k, transpose_y=True) + if attn_bias: + product += attn_bias + weights = layers.softmax(product) + if dropout_rate: + weights = layers.dropout( + weights, + dropout_prob=dropout_rate, + dropout_implementation="upscale_in_train", + is_test=False) + out = layers.matmul(weights, v) + return out + + q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value) + + if cache is not None: # use cache and concat time steps + # Since the inplace reshape in __split_heads changes the shape of k and + # v, which is the cache input for next time step, reshape the cache + # input from the previous time step first. + k = cache["k"] = layers.concat( + [layers.reshape( + cache["k"], shape=[0, 0, d_model]), k], axis=1) + v = cache["v"] = layers.concat( + [layers.reshape( + cache["v"], shape=[0, 0, d_model]), v], axis=1) + + q = __split_heads(q, n_head) + k = __split_heads(k, n_head) + v = __split_heads(v, n_head) + + ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_key, + dropout_rate) + + out = __combine_heads(ctx_multiheads) + + # Project back to the model size. + proj_out = layers.fc(input=out, + size=d_model, + num_flatten_dims=2, + param_attr=name + '_output_fc.w_0', + bias_attr=name + '_output_fc.b_0') + return proj_out + + +def positionwise_feed_forward(x, + d_inner_hid, + d_hid, + dropout_rate, + hidden_act, + name='ffn'): + """ + Position-wise Feed-Forward Networks. + This module consists of two linear transformations with a ReLU activation + in between, which is applied to each position separately and identically. + """ + hidden = layers.fc(input=x, + size=d_inner_hid, + num_flatten_dims=2, + act=hidden_act, + param_attr=name + '_fc_0.w_0', + bias_attr=name + '_fc_0.b_0') + if dropout_rate: + hidden = layers.dropout( + hidden, + dropout_prob=dropout_rate, + dropout_implementation="upscale_in_train", + is_test=False) + out = layers.fc(input=hidden, + size=d_hid, + num_flatten_dims=2, + param_attr=name + '_fc_1.w_0', + bias_attr=name + '_fc_1.b_0') + return out + + +def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0., + name=''): + """ + Add residual connection, layer normalization and droput to the out tensor + optionally according to the value of process_cmd. + This will be used before or after multi-head attention and position-wise + feed-forward networks. + """ + for cmd in process_cmd: + if cmd == "a": # add residual connection + out = out + prev_out if prev_out else out + elif cmd == "n": # add layer normalization + out = layers.layer_norm( + out, + begin_norm_axis=len(out.shape) - 1, + param_attr=fluid.ParamAttr( + name=name + '_layer_norm_scale', + initializer=fluid.initializer.Constant(1.)), + bias_attr=fluid.ParamAttr( + name=name + '_layer_norm_bias', + initializer=fluid.initializer.Constant(0.))) + elif cmd == "d": # add dropout + if dropout_rate: + out = layers.dropout( + out, + dropout_prob=dropout_rate, + dropout_implementation="upscale_in_train", + is_test=False) + return out + + +pre_process_layer = partial(pre_post_process_layer, None) +post_process_layer = pre_post_process_layer + + +def encoder_layer(enc_input, + attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + hidden_act, + preprocess_cmd="n", + postprocess_cmd="da", + name=''): + """The encoder layers that can be stacked to form a deep encoder. + This module consits of a multi-head (self) attention followed by + position-wise feed-forward networks and both the two components companied + with the post_process_layer to add residual connection, layer normalization + and droput. + """ + attn_output = multi_head_attention( + pre_process_layer( + enc_input, + preprocess_cmd, + prepostprocess_dropout, + name=name + '_pre_att'), + None, + None, + attn_bias, + d_key, + d_value, + d_model, + n_head, + attention_dropout, + name=name + '_multi_head_att') + attn_output = post_process_layer( + enc_input, + attn_output, + postprocess_cmd, + prepostprocess_dropout, + name=name + '_post_att') + ffd_output = positionwise_feed_forward( + pre_process_layer( + attn_output, + preprocess_cmd, + prepostprocess_dropout, + name=name + '_pre_ffn'), + d_inner_hid, + d_model, + relu_dropout, + hidden_act, + name=name + '_ffn') + return post_process_layer( + attn_output, + ffd_output, + postprocess_cmd, + prepostprocess_dropout, + name=name + '_post_ffn') + + +def encoder(enc_input, + attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + hidden_act, + preprocess_cmd="n", + postprocess_cmd="da", + name=''): + """ + The encoder is composed of a stack of identical layers returned by calling + encoder_layer. + """ + for i in range(n_layer): + enc_output = encoder_layer( + enc_input, + attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + hidden_act, + preprocess_cmd, + postprocess_cmd, + name=name + '_layer_' + str(i)) + enc_input = enc_output + enc_output = pre_process_layer( + enc_output, preprocess_cmd, prepostprocess_dropout, name="post_encoder") + + return enc_output diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/inputters/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/inputters/__init__.py new file mode 100644 index 00000000..beea2958 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/inputters/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: __init__.py +""" \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/inputters/data_provider.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/inputters/data_provider.py new file mode 100755 index 00000000..797ee45f --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/inputters/data_provider.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: data_provider.py +""" + +import re +import os +import types +import csv +import random +import numpy as np + + +VOC_DICT = {} + + +def load_dict(vocab_dict): + """ + load vocabulary dict + """ + idx = 0 + for line in open(vocab_dict): + line = line.strip() + VOC_DICT[line] = idx + idx += 1 + return VOC_DICT + + +def prepare_batch_data(insts, + task_name, + max_len=128, + return_attn_bias=True, + return_max_len=True, + return_num_token=False): + """ + generate self attention mask, [shape: batch_size * max_len * max_len] + """ + batch_context_ids = [inst[0] for inst in insts] + batch_context_pos_ids = [inst[1] for inst in insts] + batch_segment_ids = [inst[2] for inst in insts] + batch_label_ids = [[inst[3]] for inst in insts] + labels_list = batch_label_ids + + context_id, next_sent_context_index, context_attn_bias = \ + pad_batch_data(batch_context_ids, pad_idx=0, max_len=max_len, \ + return_next_sent_pos=True, return_attn_bias=True) + + context_pos_id = pad_batch_data( + batch_context_pos_ids, pad_idx=0, max_len=max_len, return_pos=False, return_attn_bias=False) + + context_segment_id = pad_batch_data( + batch_segment_ids, pad_idx=0, max_len=max_len, return_pos=False, return_attn_bias=False) + + if 'kn' in task_name: + batch_kn_ids = [inst[4] for inst in insts] + kn_id = pad_bath_kn_data(batch_kn_ids, pad_idx=0, max_len=max_len) + + out_list = [] + for i in range(len(insts)): + if 'kn' in task_name: + out = [context_id[i], context_pos_id[i], context_segment_id[i], context_attn_bias[i], \ + kn_id[i], labels_list[i], next_sent_context_index[i]] + else: + out = [context_id[i], context_pos_id[i], context_segment_id[i], \ + context_attn_bias[i], labels_list[i], next_sent_context_index[i]] + out_list.append(out) + return out_list + + +def pad_bath_kn_data(insts, + pad_idx=0, + max_len=128): + kn_list = [] + for inst in insts: + inst = inst[0: min(max_len, len(inst))] + kn_list.append(inst) + return kn_list + + +def pad_batch_data(insts, + pad_idx=0, + max_len=128, + return_pos=False, + return_next_sent_pos=False, + return_attn_bias=False, + return_max_len=False, + return_num_token=False): + """ + Pad the instances to the max sequence length in batch, and generate the + corresponding position data and attention bias. + """ + return_list = [] + + inst_data = np.array( + [inst + list([pad_idx] * (max_len - len(inst))) for inst in insts]) + return_list += [inst_data.astype("int64").reshape([-1, max_len, 1])] + + if return_next_sent_pos: + batch_size = inst_data.shape[0] + max_seq_len = inst_data.shape[1] + next_sent_index = np.array( + range(0, batch_size * max_seq_len, max_seq_len)).astype( + "int64").reshape(-1, 1) + return_list += [next_sent_index] + + if return_pos: + inst_pos = np.array([ + list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) + for inst in insts]) + return_list += [inst_pos.astype("int64").reshape([-1, max_len, 1])] + + if return_attn_bias: + slf_attn_bias_data = np.array([[0] * len(inst) + [-1e9] * + (max_len - len(inst)) for inst in insts]) + slf_attn_bias_data = np.tile( + slf_attn_bias_data.reshape([-1, 1, max_len]), [1, max_len, 1]) + return_list += [slf_attn_bias_data.astype("float32")] + + if return_max_len: + return_list += [max_len] + + if return_num_token: + num_token = 0 + for inst in insts: + num_token += len(inst) + return_list += [num_token] + + return return_list if len(return_list) > 1 else return_list[0] + + +def preprocessing_for_one_line(line, labels, task_name, max_seq_len=256): + """ + process text to model inputs + """ + line = line.rstrip('\n').split('\t') + label_text = line[0] + context_text = line[1] + response_text = line[2] + if 'kn' in task_name: + kn_text = "%s [SEP] %s" % (line[3], line[4]) + else: + kn_text = None + + example = InputExample(guid=0, \ + context_text=context_text, \ + response_text=response_text, \ + kn_text=kn_text, \ + label_text=label_text) + + feature = convert_single_example(0, example, labels, max_seq_len) + + instance = [feature.context_ids, feature.context_pos_ids, \ + feature.segment_ids, feature.label_ids, feature.kn_ids] + + batch_data = prepare_batch_data([instance], + task_name, + max_len=max_seq_len, + return_attn_bias=True, + return_max_len=False, + return_num_token=False) + + return batch_data + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + def __init__(self, data_dir, task_name, vocab_path, max_seq_len, do_lower_case): + self.data_dir = data_dir + self.max_seq_len = max_seq_len + self.task_name = task_name + + self.current_train_example = -1 + self.num_examples = {'train': -1, 'dev': -1, 'test': -1} + self.current_train_epoch = -1 + VOC_DICT = load_dict(vocab_path) + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_test_examples(self, data_dir): + """Gets a collection of `InputExample`s for prediction.""" + raise NotImplementedError() + + @classmethod + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + def convert_example(self, index, example, labels, max_seq_len): + """Converts a single `InputExample` into a single `InputFeatures`.""" + feature = convert_single_example(index, example, labels, max_seq_len) + return feature + + def generate_batch_data(self, + batch_data, + voc_size=-1, + mask_id=-1, + return_attn_bias=True, + return_max_len=False, + return_num_token=False): + return prepare_batch_data( + batch_data, + self.task_name, + self.max_seq_len, + return_attn_bias=True, + return_max_len=False, + return_num_token=False) + + @classmethod + def _read_data(cls, input_file): + """Reads a tab separated value file.""" + with open(input_file, "r") as f: + lines = [] + for line in f: + line = line.rstrip('\n').split('\t') + lines.append(line) + return lines + + def get_num_examples(self, phase): + """Get number of examples for train, dev or test.""" + if phase not in ['train', 'dev', 'test']: + raise ValueError("Unknown phase, which should be in ['train', 'dev', 'test'].") + return self.num_examples[phase] + + def get_train_progress(self): + """Gets progress for training phase.""" + return self.current_train_example, self.current_train_epoch + + def data_generator(self, + batch_size, + phase='train', + epoch=1, + shuffle=False): + """ + Generate data for train, dev or test. + """ + if phase == 'train': + examples = self.get_train_examples(self.data_dir) + self.num_examples['train'] = len(examples) + elif phase == 'dev': + examples = self.get_dev_examples(self.data_dir) + self.num_examples['dev'] = len(examples) + elif phase == 'test': + examples = self.get_test_examples(self.data_dir) + self.num_examples['test'] = len(examples) + else: + raise ValueError("Unknown phase, which should be in ['train', 'dev', 'test'].") + + def instance_reader(): + for epoch_index in range(epoch): + if shuffle: + random.shuffle(examples) + if phase == 'train': + self.current_train_epoch = epoch_index + for (index, example) in enumerate(examples): + if phase == 'train': + self.current_train_example = index + 1 + feature = self.convert_example( + index, example, self.get_labels(), self.max_seq_len) + if 'kn' in self.task_name: + instance = [feature.context_ids, feature.context_pos_ids, \ + feature.segment_ids, feature.label_ids, feature.kn_ids] + else: + instance = [feature.context_ids, feature.context_pos_ids, \ + feature.segment_ids, feature.label_ids] + yield instance + + def batch_reader(reader, batch_size): + batch = [] + for instance in reader(): + if len(batch) < batch_size: + batch.append(instance) + else: + yield batch + batch = [instance] + + if len(batch) > 0: + yield batch + + def wrapper(): + for batch_data in batch_reader(instance_reader, batch_size): + batch_data = self.generate_batch_data( + batch_data, + voc_size=-1, + mask_id=-1, + return_attn_bias=True, + return_max_len=False, + return_num_token=False) + yield batch_data + + return wrapper + + +class InputExample(object): + """A single training/test example""" + + def __init__(self, guid, context_text, response_text, kn_text, label_text): + self.guid = guid + self.context_text = context_text + self.response_text = response_text + self.kn_text = kn_text + self.label_text = label_text + + +class InputFeatures(object): + """input features datas""" + def __init__(self, context_ids, context_pos_ids, segment_ids, kn_ids, label_ids): + self.context_ids = context_ids + self.context_pos_ids = context_pos_ids + self.segment_ids = segment_ids + self.kn_ids = kn_ids + self.label_ids = label_ids + + +class MatchProcessor(DataProcessor): + """Processor for the Match data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_data(os.path.join(data_dir, "train.txt")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_data(os.path.join(data_dir, "dev.txt")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_data(os.path.join(data_dir, "test.txt")), "test") + + @classmethod + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = "%s-%s" % (set_type, i) + context_text = line[1] + label_text = line[0] + response_text = line[2] + if 'kn' in self.task_name: + kn_text = "%s [SEP] %s" % (line[3], line[4]) + else: + kn_text = None + examples.append( + InputExample( + guid=guid, context_text=context_text, response_text=response_text, \ + kn_text=kn_text, label_text=label_text)) + return examples + + +def convert_tokens_to_ids(tokens): + """ + convert input ids + """ + ids = [] + for token in tokens: + if token in VOC_DICT: + ids.append(VOC_DICT[token]) + else: + ids.append(VOC_DICT['[UNK]']) + return ids + + +def convert_single_example(ex_index, example, label_list, max_seq_length): + """Converts a single `InputExample` into a single `InputFeatures`.""" + label_map = {} + for (i, label) in enumerate(label_list): + label_map[label] = i + if example.context_text: + tokens_context = example.context_text + tokens_context = tokens_context.split() + else: + tokens_context = [] + + if example.response_text: + tokens_response = example.response_text + tokens_response = tokens_response.split() + else: + tokens_response = [] + + if example.kn_text: + tokens_kn = example.kn_text + tokens_kn = tokens_kn.split() + tokens_kn = tokens_kn[0: min(len(tokens_kn), max_seq_length)] + else: + tokens_kn = [] + + tokens_response = tokens_response[0: min(50, len(tokens_response))] + if len(tokens_context) > max_seq_length - len(tokens_response) - 3: + tokens_context = tokens_context[len(tokens_context) \ + + len(tokens_response) - max_seq_length + 3:] + + context_tokens = [] + segment_ids = [] + + context_tokens.append("[CLS]") + segment_ids.append(0) + context_tokens.extend(tokens_context) + segment_ids.extend([0] * len(tokens_context)) + context_tokens.append("[SEP]") + segment_ids.append(0) + + context_tokens.extend(tokens_response) + segment_ids.extend([1] * len(tokens_response)) + context_tokens.append("[SEP]") + segment_ids.append(1) + + context_ids = convert_tokens_to_ids(context_tokens) + context_pos_ids = list(range(len(context_ids))) + label_ids = label_map[example.label_text] + if tokens_kn: + kn_ids = convert_tokens_to_ids(tokens_kn) + else: + kn_ids = [] + + feature = InputFeatures( + context_ids=context_ids, + context_pos_ids=context_pos_ids, + segment_ids=segment_ids, + kn_ids = kn_ids, + label_ids=label_ids) + #if ex_index < 5: + # print("*** Example ***") + # print("guid: %s" % (example.guid)) + # print("context tokens: %s" % " ".join(context_tokens)) + # print("context_ids: %s" % " ".join([str(x) for x in context_ids])) + # print("context_pos_ids: %s" % " ".join([str(x) for x in context_pos_ids])) + # print("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + # print("kn_ids: %s" % " ".join([str(x) for x in kn_ids])) + # print("label: %s (id = %d)" % (example.label_text, label_ids)) + return feature + diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/models/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/models/__init__.py new file mode 100644 index 00000000..beea2958 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/models/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: __init__.py +""" \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/models/retrieval_model.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/models/retrieval_model.py new file mode 100755 index 00000000..4e539ae0 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/models/retrieval_model.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: retrieval_model.py +""" + +import six +import json +import numpy as np +import paddle.fluid as fluid +from source.encoders.transformer import encoder, pre_process_layer + + + +class RetrievalModel(object): + def __init__(self, + context_ids, + context_pos_ids, + context_segment_ids, + context_attn_mask, + kn_ids, + emb_size=1024, + n_layer=12, + n_head=1, + voc_size=10005, + max_position_seq_len=512, + sent_types=2, + hidden_act='relu', + prepostprocess_dropout=0.1, + attention_dropout=0.1, + weight_sharing=True): + self._emb_size = emb_size + self._n_layer = n_layer + self._n_head = n_head + self._voc_size = voc_size + self._sent_types = sent_types + self._max_position_seq_len = max_position_seq_len + self._hidden_act = hidden_act + self._weight_sharing = weight_sharing + self._prepostprocess_dropout = prepostprocess_dropout + self._attention_dropout = attention_dropout + + self._context_emb_name = "context_word_embedding" + self._memory_emb_name = "memory_word_embedding" + self._context_pos_emb_name = "context_pos_embedding" + self._context_segment_emb_name = "context_segment_embedding" + if kn_ids: + self._memory_emb_name = "memory_word_embedding" + self._build_model(context_ids, context_pos_ids, \ + context_segment_ids, context_attn_mask, kn_ids) + + def _build_memory_network(self, kn_ids, rnn_hidden_size=128): + kn_emb_out = fluid.layers.embedding( + input=kn_ids, + size=[self._voc_size, self._emb_size], + dtype='float32') + para_attr = fluid.ParamAttr(initializer=fluid.initializer.Normal(0.0, 0.02)) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Normal(0.0, 0.02)) + + fc_fw = fluid.layers.fc(input=kn_emb_out, + size=rnn_hidden_size * 3, + param_attr=para_attr, + bias_attr=False) + fc_bw = fluid.layers.fc(input=kn_emb_out, + size=rnn_hidden_size * 3, + param_attr=para_attr, + bias_attr=False) + gru_forward = fluid.layers.dynamic_gru( + input=fc_fw, + size=rnn_hidden_size, + param_attr=para_attr, + bias_attr=bias_attr, + candidate_activation='relu') + gru_backward = fluid.layers.dynamic_gru( + input=fc_bw, + size=rnn_hidden_size, + is_reverse=True, + param_attr=para_attr, + bias_attr=bias_attr, + candidate_activation='relu') + + memory_encoder_out = fluid.layers.concat( + input=[gru_forward, gru_backward], axis=1) + + memory_encoder_proj_out = fluid.layers.fc(input=memory_encoder_out, + size=256, + bias_attr=False) + return memory_encoder_out, memory_encoder_proj_out + + def _build_model(self, + context_ids, + context_pos_ids, + context_segment_ids, + context_attn_mask, + kn_ids): + + context_emb_out = fluid.layers.embedding( + input=context_ids, + size=[self._voc_size, self._emb_size], + param_attr=fluid.ParamAttr(name=self._context_emb_name), + is_sparse=False) + + context_position_emb_out = fluid.layers.embedding( + input=context_pos_ids, + size=[self._max_position_seq_len, self._emb_size], + param_attr=fluid.ParamAttr(name=self._context_pos_emb_name), ) + + context_segment_emb_out = fluid.layers.embedding( + input=context_segment_ids, + size=[self._sent_types, self._emb_size], + param_attr=fluid.ParamAttr(name=self._context_segment_emb_name), ) + + context_emb_out = context_emb_out + context_position_emb_out + context_emb_out = context_emb_out + context_segment_emb_out + + context_emb_out = pre_process_layer( + context_emb_out, 'nd', self._prepostprocess_dropout, name='context_pre_encoder') + + n_head_context_attn_mask = fluid.layers.stack( + x=[context_attn_mask] * self._n_head, axis=1) + + n_head_context_attn_mask.stop_gradient = True + + self._context_enc_out = encoder( + enc_input=context_emb_out, + attn_bias=n_head_context_attn_mask, + n_layer=self._n_layer, + n_head=self._n_head, + d_key=self._emb_size // self._n_head, + d_value=self._emb_size // self._n_head, + d_model=self._emb_size, + d_inner_hid=self._emb_size * 4, + prepostprocess_dropout=self._prepostprocess_dropout, + attention_dropout=self._attention_dropout, + relu_dropout=0, + hidden_act=self._hidden_act, + preprocess_cmd="an", + postprocess_cmd="dan", + name='context_encoder') + + if kn_ids: + self.memory_encoder_out, self.memory_encoder_proj_out = \ + self._build_memory_network(kn_ids) + + def get_context_output(self, context_next_sent_index, task_name): + if "kn" in task_name: + cls_feats = self.get_context_response_memory(context_next_sent_index) + else: + cls_feats = self.get_pooled_output(context_next_sent_index) + return cls_feats + + def get_context_response_memory(self, context_next_sent_index): + context_out = self.get_pooled_output(context_next_sent_index) + kn_context = self.attention(context_out, \ + self.memory_encoder_out, self.memory_encoder_proj_out) + cls_feats = fluid.layers.concat(input=[context_out, kn_context], axis=1) + return cls_feats + + def attention(self, hidden_mem, encoder_vec, encoder_vec_proj): + concated = fluid.layers.sequence_expand( + x=hidden_mem, y=encoder_vec_proj) + + concated = encoder_vec_proj + concated + concated = fluid.layers.tanh(x=concated) + attention_weights = fluid.layers.fc(input=concated, + size=1, + act=None, + bias_attr=False) + attention_weights = fluid.layers.sequence_softmax( + input=attention_weights) + weigths_reshape = fluid.layers.reshape(x=attention_weights, shape=[-1]) + scaled = fluid.layers.elementwise_mul( + x=encoder_vec, y=weigths_reshape, axis=0) + context = fluid.layers.sequence_pool(input=scaled, pool_type='sum') + return context + + def get_sequence_output(self): + return (self._context_enc_out, self._response_enc_out) + + def get_pooled_output(self, context_next_sent_index): + context_out = self.get_pooled(context_next_sent_index) + return context_out + + def get_pooled(self, next_sent_index): + """Get the first feature of each sequence for classification""" + reshaped_emb_out = fluid.layers.reshape( + x=self._context_enc_out, shape=[-1, self._emb_size], inplace=True) + next_sent_index = fluid.layers.cast(x=next_sent_index, dtype='int32') + next_sent_feat = fluid.layers.gather( + input=reshaped_emb_out, index=next_sent_index) + next_sent_feat = fluid.layers.fc( + input=next_sent_feat, + size=self._emb_size, + act="tanh", + param_attr=fluid.ParamAttr( + name="pooled_fc.w_0", + initializer=fluid.initializer.TruncatedNormal(scale=0.02)), + bias_attr="pooled_fc.b_0") + return next_sent_feat + + + def get_pooled_output_no_share(self, context_next_sent_index, response_next_sent_index): + """get pooled embedding""" + self._context_reshaped_emb_out = fluid.layers.reshape( + x=self._context_enc_out, shape=[-1, self._emb_size], inplace=True) + context_next_sent_index = fluid.layers.cast(x=context_next_sent_index, dtype='int32') + context_out = fluid.layers.gather( + input=self._context_reshaped_emb_out, index=context_next_sent_index) + context_out = fluid.layers.fc( + input=context_out, + size=self._emb_size, + act="tanh", + param_attr=fluid.ParamAttr( + name="pooled_context_fc.w_0", + initializer=fluid.initializer.TruncatedNormal(scale=0.02)), + bias_attr="pooled_context_fc.b_0") + + self._response_reshaped_emb_out = fluid.layers.reshape( + x=self._response_enc_out, shape=[-1, self._emb_size], inplace=True) + response_next_sent_index = fluid.layers.cast(x=response_next_sent_index, dtype='int32') + response_next_sent_feat = fluid.layers.gather( + input=self._response_reshaped_emb_out, index=response_next_sent_index) + response_next_sent_feat = fluid.layers.fc( + input=response_next_sent_feat, + size=self._emb_size, + act="tanh", + param_attr=fluid.ParamAttr( + name="pooled_response_fc.w_0", + initializer=fluid.initializer.TruncatedNormal(scale=0.02)), + bias_attr="pooled_response_fc.b_0") + + return context_out, response_next_sent_feat + + diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/utils/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/utils/__init__.py new file mode 100644 index 00000000..beea2958 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/utils/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: __init__.py +""" \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/utils/utils.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/utils/utils.py new file mode 100755 index 00000000..c54fdcb1 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/source/utils/utils.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: utils.py +""" + +from __future__ import print_function + +import os +import six +import ast +import copy + +import numpy as np +import paddle.fluid as fluid + + +def init_checkpoint(exe, init_checkpoint_path, main_program): + assert os.path.exists( + init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path + fluid.io.load_persistables( + exe, init_checkpoint_path, main_program=main_program) + print("Load model from {}".format(init_checkpoint_path)) + + +def init_pretraining_params(exe, pretraining_params_path, main_program): + assert os.path.exists(pretraining_params_path + ), "[%s] cann't be found." % pretraining_params_path + + def existed_params(var): + if not isinstance(var, fluid.framework.Parameter): + return False + return os.path.exists(os.path.join(pretraining_params_path, var.name)) + + fluid.io.load_vars( + exe, + pretraining_params_path, + main_program=main_program, + predicate=existed_params) + print("Load pretraining parameters from {}".format(pretraining_params_path)) + diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/__init__.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/__init__.py new file mode 100644 index 00000000..beea2958 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: __init__.py +""" \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/build_candidate_set_from_corpus.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/build_candidate_set_from_corpus.py new file mode 100644 index 00000000..57500bc1 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/build_candidate_set_from_corpus.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: build_candidate_set_from_corpus.py +""" + +from __future__ import print_function +import sys +import json +import random +import collections + +reload(sys) +sys.setdefaultencoding('utf8') + + +def build_candidate_set_from_corpus(corpus_file, candidate_set_file): + """ + build candidate set from corpus + """ + candidate_set_gener = {} + candidate_set_mater = {} + candidate_set_list = [] + slot_dict = {"topic_a": 1, "topic_b": 1} + with open(corpus_file, 'r') as f: + for i, line in enumerate(f): + conversation = json.loads(line.strip(), encoding="utf-8", \ + object_pairs_hook=collections.OrderedDict) + + chat_path = conversation["goal"] + knowledge = conversation["knowledge"] + session = conversation["conversation"] + + topic_a = chat_path[0][1] + topic_b = chat_path[0][2] + domain_a = None + domain_b = None + cover_att_list = [[["topic_a", topic_a], ["topic_b", topic_b]]] * len(session) + for j, [s, p, o] in enumerate(knowledge): + p_key = "" + if topic_a.replace(' ', '') == s.replace(' ', ''): + p_key = "topic_a_" + p.replace(' ', '') + if u"领域" == p: + domain_a = o + elif topic_b.replace(' ', '') == s.replace(' ', ''): + p_key = "topic_b_" + p.replace(' ', '') + if u"领域" == p: + domain_b = o + + for k, utterance in enumerate(session): + if k % 2 == 1: continue + if o in utterance and o != topic_a and o != topic_b and p_key != "": + cover_att_list[k].append([p_key, o]) + + slot_dict[p_key] = 1 + + assert domain_a is not None and domain_b is not None + + for j, utterance in enumerate(session): + if j % 2 == 1: continue + key = '_'.join([domain_a, domain_b, str(j)]) + + cover_att = sorted(cover_att_list[j], lambda x, y: cmp(len(x[1]), len(y[1])), reverse=True) + + utterance_gener = utterance + for [p_key, o] in cover_att: + utterance_gener = utterance_gener.replace(o, p_key) + + if "topic_a_topic_a_" not in utterance_gener and \ + "topic_a_topic_b_" not in utterance_gener and \ + "topic_b_topic_a_" not in utterance_gener and \ + "topic_b_topic_b_" not in utterance_gener: + if key in candidate_set_gener: + candidate_set_gener[key].append(utterance_gener) + else: + candidate_set_gener[key] = [utterance_gener] + + utterance_mater = utterance + for [p_key, o] in [["topic_a", topic_a], ["topic_b", topic_b]]: + utterance_mater = utterance_mater.replace(o, p_key) + + if key in candidate_set_mater: + candidate_set_mater[key].append(utterance_mater) + else: + candidate_set_mater[key] = [utterance_mater] + + candidate_set_list.append(utterance_mater) + + fout = open(candidate_set_file, 'w') + fout.write(json.dumps(candidate_set_gener, ensure_ascii=False, encoding="utf-8") + "\n") + fout.write(json.dumps(candidate_set_mater, ensure_ascii=False, encoding="utf-8") + "\n") + fout.write(json.dumps(candidate_set_list, ensure_ascii=False, encoding="utf-8") + "\n") + fout.write(json.dumps(slot_dict, ensure_ascii=False, encoding="utf-8")) + fout.close() + + +def main(): + """ + main + """ + build_candidate_set_from_corpus(sys.argv[1], sys.argv[2]) + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") \ No newline at end of file diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/build_dict.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/build_dict.py new file mode 100755 index 00000000..e9ba7924 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/build_dict.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: build_dict.py +""" + +from __future__ import print_function +import sys + +reload(sys) +sys.setdefaultencoding('utf8') + + +def build_dict(corpus_file, dict_file): + """ + build words dict + """ + dict = {} + max_frequency = 1 + for line in open(corpus_file, 'r'): + conversation = line.strip().split('\t') + for i in range(1, len(conversation), 1): + words = conversation[i].split(' ') + for word in words: + if word in dict: + dict[word] = dict[word] + 1 + if dict[word] > max_frequency: + max_frequency = dict[word] + else: + dict[word] = 1 + + dict["[PAD]"] = max_frequency + 4 + dict["[UNK]"] = max_frequency + 3 + dict["[CLS]"] = max_frequency + 2 + dict["[SEP]"] = max_frequency + 1 + + words = sorted(dict.items(), lambda x, y: cmp(x[1], y[1]), reverse=True) + + fout = open(dict_file, 'w') + for word, frequency in words: + fout.write(word + '\n') + + fout.close() + + +def main(): + """ + main + """ + if len(sys.argv) < 3: + print("Usage: " + sys.argv[0] + " corpus_file dict_file") + exit() + + build_dict(sys.argv[1], sys.argv[2]) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/construct_candidate.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/construct_candidate.py new file mode 100644 index 00000000..722848f9 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/construct_candidate.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: construct_candidate.py +""" + +from __future__ import print_function +import sys +import json +import random +import collections + +reload(sys) +sys.setdefaultencoding('utf8') + + +def load_candidate_set(candidate_set_file): + """ + load candidate set + """ + candidate_set = [] + for line in open(candidate_set_file): + candidate_set.append(json.loads(line.strip(), encoding="utf-8")) + + return candidate_set + + +def candidate_slection(candidate_set, knowledge_dict, slot_dict, candidate_num=10): + """ + candidate slection + """ + random.shuffle(candidate_set) + candidate_legal = [] + for candidate in candidate_set: + is_legal = True + for slot in slot_dict: + if slot in ["topic_a", "topic_b"]: + continue + if slot in candidate: + if slot not in knowledge_dict: + is_legal = False + break + w_ = random.choice(knowledge_dict[slot]) + candidate = candidate.replace(slot, w_) + + for slot in ["topic_a", "topic_b"]: + if slot in candidate: + if slot not in knowledge_dict: + is_legal = False + break + w_ = random.choice(knowledge_dict[slot]) + candidate = candidate.replace(slot, w_) + + if is_legal and candidate not in candidate_legal: + candidate_legal.append(candidate) + + if len(candidate_legal) >= candidate_num: + break + + return candidate_legal + + +def get_candidate_for_conversation(conversation, candidate_set, candidate_num=10): + """ + get candidate for conversation + """ + candidate_set_gener, candidate_set_mater, candidate_set_list, slot_dict = candidate_set + + chat_path = conversation["goal"] + knowledge = conversation["knowledge"] + history = conversation["history"] + + topic_a = chat_path[0][1] + topic_b = chat_path[0][2] + domain_a = None + domain_b = None + knowledge_dict = {"topic_a":[topic_a], "topic_b":[topic_b]} + for i, [s, p, o] in enumerate(knowledge): + p_key = "" + if topic_a.replace(' ', '') == s.replace(' ', ''): + p_key = "topic_a_" + p.replace(' ', '') + if u"领域" == p: + domain_a = o + elif topic_b.replace(' ', '') == s.replace(' ', ''): + p_key = "topic_b_" + p.replace(' ', '') + if u"领域" == p: + domain_b = o + + if p_key == "": + continue + + if p_key in knowledge_dict: + knowledge_dict[p_key].append(o) + else: + knowledge_dict[p_key] = [o] + + assert domain_a is not None and domain_b is not None + + key = '_'.join([domain_a, domain_b, str(len(history))]) + + candidate_legal = [] + if key in candidate_set_gener: + candidate_legal.extend(candidate_slection(candidate_set_gener[key], + knowledge_dict, slot_dict, + candidate_num = candidate_num - len(candidate_legal))) + + if len(candidate_legal) < candidate_num and key in candidate_set_mater: + candidate_legal.extend(candidate_slection(candidate_set_mater[key], + knowledge_dict, slot_dict, + candidate_num = candidate_num - len(candidate_legal))) + + if len(candidate_legal) < candidate_num: + candidate_legal.extend(candidate_slection(candidate_set_list, + knowledge_dict, slot_dict, + candidate_num = candidate_num - len(candidate_legal))) + + return candidate_legal + + +def construct_candidate_for_corpus(corpus_file, candidate_set_file, candidate_file, candidate_num=10): + """ + construct candidate for corpus + + case of data in corpus_file: + { + "goal": [["START", "休 · 劳瑞", "蕾切儿 · 哈伍德"]], + "knowledge": [["休 · 劳瑞", "评论", "完美 的 男人"]], + "history": ["你 对 明星 有没有 到 迷恋 的 程度 呢 ？", + "一般 吧 ， 毕竟 年纪 不 小 了 ， 只是 追星 而已 。"] + } + + case of data in candidate_file: + { + "goal": [["START", "休 · 劳瑞", "蕾切儿 · 哈伍德"]], + "knowledge": [["休 · 劳瑞", "评论", "完美 的 男人"]], + "history": ["你 对 明星 有没有 到 迷恋 的 程度 呢 ？", + "一般 吧 ， 毕竟 年纪 不 小 了 ， 只是 追星 而已 。"], + "candidate": ["我 说 的 是 休 · 劳瑞 。", + "我 说 的 是 休 · 劳瑞 。"] + } + """ + candidate_set = load_candidate_set(candidate_set_file) + fout_text = open(candidate_file, 'w') + with open(corpus_file, 'r') as f: + for i, line in enumerate(f): + conversation = json.loads(line.strip(), encoding="utf-8", \ + object_pairs_hook=collections.OrderedDict) + candidates = get_candidate_for_conversation(conversation, + candidate_set, + candidate_num=candidate_num) + conversation["candidate"] = candidates + + conversation = json.dumps(conversation, ensure_ascii=False, encoding="utf-8") + fout_text.write(conversation + "\n") + + fout_text.close() + + +def main(): + """ + main + """ + construct_candidate_for_corpus(sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_client.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_client.py new file mode 100755 index 00000000..d55a3859 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_client.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: conversation_client.py +""" + +from __future__ import print_function +import sys +import socket +reload(sys) +sys.setdefaultencoding('utf8') + +SERVER_IP = "127.0.0.1" +SERVER_PORT = 8601 + +def conversation_client(text): + """ + conversation_client + """ + mysocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + mysocket.connect((SERVER_IP, SERVER_PORT)) + + mysocket.sendall(text.encode()) + result = mysocket.recv(4096).decode() + + mysocket.close() + + return result + + +def main(): + """ + main + """ + if len(sys.argv) < 2: + print("Usage: " + sys.argv[0] + " eval_file") + exit() + + for line in open(sys.argv[1]): + response = conversation_client(line.strip()) + print(response) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_server.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_server.py new file mode 100644 index 00000000..2ee32679 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_server.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: conversation_server.py +""" + +from __future__ import print_function + +import sys +sys.path.append("../") +import socket +from thread import start_new_thread +from tools.conversation_strategy import load +from tools.conversation_strategy import predict +reload(sys) +sys.setdefaultencoding('utf8') + +SERVER_IP = "127.0.0.1" +SERVER_PORT = 8601 + +print("starting conversation server ...") +print("binding socket ...") +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +# Bind socket to local host and port +try: + s.bind((SERVER_IP, SERVER_PORT)) +except socket.error as msg: + print("Bind failed. Error Code : " + str(msg[0]) + " Message " + msg[1]) + exit() +# Start listening on socket +s.listen(10) +print("bind socket success !") + +print("loading model...") +model = load() +print("load model success !") + +print("start conversation server success !") + + +def clientthread(conn, addr): + """ + client thread + """ + logstr = "addr:" + addr[0] + "_" + str(addr[1]) + try: + # Receiving from client + param = conn.recv(4096).decode() + logstr += "\tparam:" + param + if param is not None: + response = predict(model, param.strip()) + logstr += "\tresponse:" + response + conn.sendall(response.encode()) + conn.close() + print(logstr + "\n") + except Exception as e: + print(logstr + "\n", e) + + +while True: + conn, addr = s.accept() + start_new_thread(clientthread, (conn, addr)) +s.close() diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_strategy.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_strategy.py new file mode 100644 index 00000000..2f8aad5d --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/conversation_strategy.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: conversation_strategy.py +""" + +from __future__ import print_function + +import sys +sys.path.append("../") +import interact +from tools.convert_conversation_corpus_to_model_text import preprocessing_for_one_conversation +from tools.construct_candidate import load_candidate_set + +reload(sys) +sys.setdefaultencoding('utf8') + + +def load(): + """ + load + """ + return interact.load_model(), load_candidate_set("../data/candidate_set.txt") + + +def predict(model, text): + """ + predict + """ + model, candidate_set = model + model_text, candidates = \ + preprocessing_for_one_conversation(text.strip(), + candidate_set=candidate_set, + candidate_num=50, + use_knowledge=True, + topic_generalization=True, + for_predict=True) + + for i, text_ in enumerate(model_text): + score = interact.predict(model, text_, task_name="match_kn_gene") + candidates[i] = [candidates[i], score] + + candidate_legal = sorted(candidates, key=lambda item: item[1], reverse=True) + return candidate_legal[0][0] + + +def main(): + """ + main + """ + model = load() + for line in sys.stdin: + response = predict(model, line.strip()) + print(response) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/convert_conversation_corpus_to_model_text.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/convert_conversation_corpus_to_model_text.py new file mode 100644 index 00000000..a22e8dc3 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/convert_conversation_corpus_to_model_text.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: convert_conversation_corpus_to_model_text.py +""" + +from __future__ import print_function +import sys +sys.path.append("./") +import re +import json +import collections +from tools.construct_candidate import get_candidate_for_conversation + +reload(sys) +sys.setdefaultencoding('utf8') + +def parser_char_for_word(word): + """ + parser char for word + """ + if word.isdigit(): + word = word.decode('utf8') + for i in range(len(word)): + if word[i] >= u'\u4e00' and word[i] <= u'\u9fa5': + word_out = " ".join([t.encode('utf8') for t in word]) + word_out = re.sub(" +", " ", word_out) + return word_out + return word.encode('utf8') + + +def parser_char_for_text(text): + """ + parser char for text + """ + words = text.strip().split() + for i, word in enumerate(words): + words[i] = parser_char_for_word(word) + return re.sub(" +", " ", ' '.join(words)) + + +def topic_generalization_for_text(text, topic_list): + """ + topic generalization for text + """ + for key, value in topic_list: + text = text.replace(value, key) + + return text + + +def topic_generalization_for_list(text_list, topic_list): + """ + topic generalization for list + """ + for i, text in enumerate(text_list): + text_list[i] = topic_generalization_for_text(text, topic_list) + + return text_list + + +def preprocessing_for_one_conversation(text, \ + candidate_set=None, \ + candidate_num=10, \ + use_knowledge=True, \ + topic_generalization=False, \ + for_predict=True): + """ + preprocessing for one conversation + """ + + conversation = json.loads(text.strip(), encoding="utf-8", \ + object_pairs_hook=collections.OrderedDict) + + goal = conversation["goal"] + knowledge = conversation["knowledge"] + history = conversation["history"] + if not for_predict: + response = conversation["response"] + + + topic_a = goal[0][1] + topic_b = goal[0][2] + for i, [s, p, o] in enumerate(knowledge): + if u"领域" == p: + if topic_a == s: + domain_a = o + elif topic_b == s: + domain_b = o + + topic_dict = {} + if u"电影" == domain_a: + topic_dict["video_topic_a"] = topic_a + else: + topic_dict["person_topic_a"] = topic_a + + if u"电影" == domain_b: + topic_dict["video_topic_b"] = topic_b + else: + topic_dict["person_topic_b"] = topic_b + + if "candidate" in conversation: + candidates = conversation["candidate"] + else: + assert candidate_num > 0 and candidate_set is not None + candidates = get_candidate_for_conversation(conversation, + candidate_set, + candidate_num=candidate_num) + + if topic_generalization: + topic_list = sorted(topic_dict.items(), key=lambda item: len(item[1]), reverse=True) + + goal = [topic_generalization_for_list(spo, topic_list) for spo in goal] + + knowledge = [topic_generalization_for_list(spo, topic_list) for spo in knowledge] + + history = [topic_generalization_for_text(utterance, topic_list) + for utterance in history] + + for i, candidate in enumerate(candidates): + candidates[i] = topic_generalization_for_text(candidate, topic_list) + + if not for_predict: + response = topic_generalization_for_text(response, topic_list) + + goal = ' [PATH_SEP] '.join([parser_char_for_text(' '.join(spo)) + for spo in goal]) + knowledge = ' [KN_SEP] '.join([parser_char_for_text(' '.join(spo)) + for spo in knowledge]) + history = ' [INNER_SEP] '.join([parser_char_for_text(utterance) + for utterance in history]) \ + if len(history) > 0 else '[START]' + + model_text = [] + + for candidate in candidates: + candidate = parser_char_for_text(candidate) + if use_knowledge: + text_ = '\t'.join(["0", history, candidate, goal, knowledge]) + else: + text_ = '\t'.join(["0", history, candidate]) + + text_ = re.sub(" +", " ", text_) + model_text.append(text_) + + if not for_predict: + candidates.append(response) + response = parser_char_for_text(response) + if use_knowledge: + text_ = '\t'.join(["1", history, response, goal, knowledge]) + else: + text_ = '\t'.join(["1", history, response]) + + text_ = re.sub(" +", " ", text_) + model_text.append(text_) + + return model_text, candidates + + +def convert_conversation_corpus_to_model_text(corpus_file, + text_file, + use_knowledge=True, + topic_generalization=False, + for_predict=True): + """ + convert conversation corpus to model text + """ + fout_text = open(text_file, 'w') + with open(corpus_file, 'r') as f: + for i, line in enumerate(f): + model_text, _ = preprocessing_for_one_conversation( + line.strip(), + candidate_set=None, + candidate_num=0, + use_knowledge=use_knowledge, + topic_generalization=topic_generalization, + for_predict=for_predict) + + for text in model_text: + fout_text.write(text + "\n") + + fout_text.close() + + +def main(): + """ + main + """ + convert_conversation_corpus_to_model_text(sys.argv[1], + sys.argv[2], + int(sys.argv[3]) > 0, + int(sys.argv[4]) > 0, + int(sys.argv[5]) > 0) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/convert_session_to_sample.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/convert_session_to_sample.py new file mode 100755 index 00000000..6d7d20d2 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/convert_session_to_sample.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: convert_session_to_sample.py +""" + +from __future__ import print_function +import sys +import json +import collections + +reload(sys) +sys.setdefaultencoding('utf8') + + +def convert_session_to_sample(session_file, sample_file): + """ + convert_session_to_sample + """ + fout = open(sample_file, 'w') + with open(session_file, 'r') as f: + for i, line in enumerate(f): + session = json.loads(line.strip(), encoding="utf-8", \ + object_pairs_hook=collections.OrderedDict) + conversation = session["conversation"] + + for j in range(0, len(conversation), 2): + sample = collections.OrderedDict() + sample["goal"] = session["goal"] + sample["knowledge"] = session["knowledge"] + sample["history"] = conversation[:j] + sample["response"] = conversation[j] + + sample = json.dumps(sample, ensure_ascii=False, encoding="utf-8") + + fout.write(sample + "\n") + + fout.close() + + +def main(): + """ + main + """ + convert_session_to_sample(sys.argv[1], sys.argv[2]) + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/eval.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/eval.py new file mode 100755 index 00000000..e7077e65 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/eval.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: eval.py +""" + +from __future__ import print_function + +import sys +import math +from collections import Counter + +reload(sys) +sys.setdefaultencoding('utf8') + +if len(sys.argv) < 2: + print("Usage: " + sys.argv[0] + " eval_file") + print("eval file format: pred_response \t gold_response") + exit() + +def get_dict(tokens, ngram, gdict=None): + """ + get_dict + """ + token_dict = {} + if gdict is not None: + token_dict = gdict + tlen = len(tokens) + for i in range(0, tlen - ngram + 1): + ngram_token = "".join(tokens[i:(i + ngram)]) + if token_dict.get(ngram_token) is not None: + token_dict[ngram_token] += 1 + else: + token_dict[ngram_token] = 1 + return token_dict + + +def count(pred_tokens, gold_tokens, ngram, result): + """ + count + """ + cover_count, total_count = result + pred_dict = get_dict(pred_tokens, ngram) + gold_dict = get_dict(gold_tokens, ngram) + cur_cover_count = 0 + cur_total_count = 0 + for token, freq in pred_dict.items(): + if gold_dict.get(token) is not None: + gold_freq = gold_dict[token] + cur_cover_count += min(freq, gold_freq) + cur_total_count += freq + result[0] += cur_cover_count + result[1] += cur_total_count + + +def calc_bp(pair_list): + """ + calc_bp + """ + c_count = 0.0 + r_count = 0.0 + for pair in pair_list: + pred_tokens, gold_tokens = pair + c_count += len(pred_tokens) + r_count += len(gold_tokens) + bp = 1 + if c_count < r_count: + bp = math.exp(1 - r_count / c_count) + return bp + + +def calc_cover_rate(pair_list, ngram): + """ + calc_cover_rate + """ + result = [0.0, 0.0] # [cover_count, total_count] + for pair in pair_list: + pred_tokens, gold_tokens = pair + count(pred_tokens, gold_tokens, ngram, result) + cover_rate = result[0] / result[1] + return cover_rate + + +def calc_bleu(pair_list): + """ + calc_bleu + """ + bp = calc_bp(pair_list) + cover_rate1 = calc_cover_rate(pair_list, 1) + cover_rate2 = calc_cover_rate(pair_list, 2) + cover_rate3 = calc_cover_rate(pair_list, 3) + bleu1 = 0 + bleu2 = 0 + bleu3 = 0 + if cover_rate1 > 0: + bleu1 = bp * math.exp(math.log(cover_rate1)) + if cover_rate2 > 0: + bleu2 = bp * math.exp((math.log(cover_rate1) + math.log(cover_rate2)) / 2) + if cover_rate3 > 0: + bleu3 = bp * math.exp((math.log(cover_rate1) + math.log(cover_rate2) + math.log(cover_rate3)) / 3) + return [bleu1, bleu2] + + +def calc_distinct_ngram(pair_list, ngram): + """ + calc_distinct_ngram + """ + ngram_total = 0.0 + ngram_distinct_count = 0.0 + pred_dict = {} + for predict_tokens, _ in pair_list: + get_dict(predict_tokens, ngram, pred_dict) + for key, freq in pred_dict.items(): + ngram_total += freq + ngram_distinct_count += 1 + #if freq == 1: + # ngram_distinct_count += freq + return ngram_distinct_count / ngram_total + + +def calc_distinct(pair_list): + """ + calc_distinct + """ + distinct1 = calc_distinct_ngram(pair_list, 1) + distinct2 = calc_distinct_ngram(pair_list, 2) + return [distinct1, distinct2] + + +def calc_f1(data): + """ + calc_f1 + """ + golden_char_total = 0.0 + pred_char_total = 0.0 + hit_char_total = 0.0 + for response, golden_response in data: + golden_response = "".join(golden_response).decode("utf8") + response = "".join(response).decode("utf8") + #golden_response = "".join(golden_response) + #response = "".join(response) + common = Counter(response) & Counter(golden_response) + hit_char_total += sum(common.values()) + golden_char_total += len(golden_response) + pred_char_total += len(response) + p = hit_char_total / pred_char_total + r = hit_char_total / golden_char_total + f1 = 2 * p * r / (p + r) + return f1 + + +eval_file = sys.argv[1] +sents = [] +for line in open(eval_file): + tk = line.strip().split("\t") + if len(tk) < 2: + continue + pred_tokens = tk[0].strip().split(" ") + gold_tokens = tk[1].strip().split(" ") + sents.append([pred_tokens, gold_tokens]) +# calc f1 +f1 = calc_f1(sents) +# calc bleu +bleu1, bleu2 = calc_bleu(sents) +# calc distinct +distinct1, distinct2 = calc_distinct(sents) + +output_str = "F1: %.2f%%\n" % (f1 * 100) +output_str += "BLEU1: %.3f%%\n" % bleu1 +output_str += "BLEU2: %.3f%%\n" % bleu2 +output_str += "DISTINCT1: %.3f%%\n" % distinct1 +output_str += "DISTINCT2: %.3f%%\n" % distinct2 +sys.stdout.write(output_str) diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/extract_predict_utterance.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/extract_predict_utterance.py new file mode 100755 index 00000000..99e485b9 --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/tools/extract_predict_utterance.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved +# +################################################################################ +""" +File: extract_predict_utterance.py +""" + +from __future__ import print_function + +import sys +import json +import collections + +reload(sys) +sys.setdefaultencoding('utf8') + + +def extract_predict_utterance(sample_file, score_file, output_file): + """ + convert_result_for_eval + """ + sample_list = [line.strip() for line in open(sample_file, 'r')] + score_list = [line.strip() for line in open(score_file, 'r')] + + fout = open(output_file, 'w') + index = 0 + for i, sample in enumerate(sample_list): + sample = json.loads(sample, encoding="utf-8", \ + object_pairs_hook=collections.OrderedDict) + + candidates = sample["candidate"] + scores = score_list[index: index + len(candidates)] + + pridict = candidates[0] + max_score = float(scores[0]) + for j, score in enumerate(scores): + score = float(score) + if score > max_score: + pridict = candidates[j] + max_score = score + + if "response" in sample: + response = sample["response"] + fout.write(pridict + "\t" + response + "\n") + else: + fout.write(pridict + "\n") + + index = index + len(candidates) + + fout.close() + + +def main(): + """ + main + """ + extract_predict_utterance(sys.argv[1], + sys.argv[2], + sys.argv[3]) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nExited from the program ealier!") diff --git a/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/train.py b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/train.py new file mode 100644 index 00000000..243b29fb --- /dev/null +++ b/PaddleNLP/Research/ACL2019-DuConv/retrieval_paddle/train.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +###################################################################### +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +""" +File: train.py +""" + +import os +import time +import numpy as np +import multiprocessing + +import paddle +import paddle.fluid as fluid +import paddle.fluid.framework as framework +from paddle.fluid.executor import Executor + +import source.inputters.data_provider as reader +from source.models.retrieval_model import RetrievalModel +from args import base_parser +from args import print_arguments + + +def create_model(args, num_labels, is_prediction=False): + context_ids = fluid.layers.data(name='context_ids', shape=[-1, args.max_seq_len, 1], dtype='int64', lod_level=0) + context_pos_ids = fluid.layers.data(name='context_pos_ids', shape=[-1, args.max_seq_len, 1], dtype='int64', lod_level=0) + context_segment_ids = fluid.layers.data(name='context_segment_ids', shape=[-1, args.max_seq_len, 1], dtype='int64', lod_level=0) + context_attn_mask = fluid.layers.data(name='context_attn_mask', shape=[-1, args.max_seq_len, args.max_seq_len], dtype='float', lod_level=0) + labels = fluid.layers.data(name='labels', shape=[1], dtype='int64', lod_level=0) + context_next_sent_index = fluid.layers.data(name='context_next_sent_index', shape=[1], dtype='int64', lod_level=0) + + if "kn" in args.task_name: + kn_ids = fluid.layers.data(name='kn_ids', shape=[1], dtype='int64', lod_level=1) + feed_order = ["context_ids", "context_pos_ids", "context_segment_ids", "context_attn_mask", "kn_ids", "labels", "context_next_sent_index"] + else: + kn_ids = None + feed_order = ["context_ids", "context_pos_ids", "context_segment_ids", "context_attn_mask", "labels", "context_next_sent_index"] + + if is_prediction: + dropout_prob = 0.1 + attention_dropout = 0.1 + prepostprocess_dropout = 0.1 + else: + dropout_prob = 0.0 + attention_dropout = 0.0 + prepostprocess_dropout = 0.0 + + retrieval_model = RetrievalModel( + context_ids=context_ids, + context_pos_ids=context_pos_ids, + context_segment_ids=context_segment_ids, + context_attn_mask=context_attn_mask, + kn_ids=kn_ids, + emb_size=256, + n_layer=4, + n_head=8, + voc_size=args.voc_size, + max_position_seq_len=args.max_seq_len, + hidden_act="gelu", + attention_dropout=attention_dropout, + prepostprocess_dropout=prepostprocess_dropout) + + context_cls = retrieval_model.get_context_output(context_next_sent_index, args.task_name) + context_cls = fluid.layers.dropout( + x=context_cls, + dropout_prob=dropout_prob, + dropout_implementation="upscale_in_train") + + cls_feats = context_cls + logits = fluid.layers.fc( + input=cls_feats, + size=num_labels, + param_attr=fluid.ParamAttr( + name="cls_out_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02)), + bias_attr=fluid.ParamAttr( + name="cls_out_b", initializer=fluid.initializer.Constant(0.))) + + ce_loss, predict = fluid.layers.softmax_with_cross_entropy( + logits=logits, label=labels, return_softmax=True) + loss = fluid.layers.reduce_mean(input=ce_loss) + + num_seqs = fluid.layers.create_tensor(dtype='int64') + accuracy = fluid.layers.accuracy(input=predict, label=labels, total=num_seqs) + + loss.persistable = True + predict.persistable = True + accuracy.persistable = True + num_seqs.persistable = True + + return feed_order, loss, predict, accuracy, num_seqs + + +def main(args): + + task_name = args.task_name.lower() + processor = reader.MatchProcessor(data_dir=args.data_dir, + task_name=task_name, + vocab_path=args.vocab_path, + max_seq_len=args.max_seq_len, + do_lower_case=args.do_lower_case) + + args.voc_size = len(open(args.vocab_path, 'r').readlines()) + num_labels = len(processor.get_labels()) + train_data_generator = processor.data_generator( + batch_size=args.batch_size, + phase='train', + epoch=args.epoch, + shuffle=True) + num_train_examples = processor.get_num_examples(phase='train') + dev_data_generator = processor.data_generator( + batch_size=args.batch_size, + phase='dev', + epoch=1, + shuffle=False) + num_dev_examples = processor.get_num_examples(phase='dev') + + if args.use_cuda: + place = fluid.CUDAPlace(0) + dev_count = fluid.core.get_cuda_device_count() + else: + place = fluid.CPUPlace() + dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + + max_train_steps = args.epoch * num_train_examples // args.batch_size + warmup_steps = int(max_train_steps * args.warmup_proportion) + + train_program = fluid.Program() + train_startup = fluid.Program() + with fluid.program_guard(train_program, train_startup): + with fluid.unique_name.guard(): + feed_order, loss, predict, accuracy, num_seqs = \ + create_model(args, num_labels, \ + is_prediction=False) + lr_decay = fluid.layers.learning_rate_scheduler.noam_decay(256, warmup_steps) + with fluid.default_main_program()._lr_schedule_guard(): + learning_rate = lr_decay * args.learning_rate + optimizer = fluid.optimizer.Adam( + learning_rate=learning_rate) + optimizer.minimize(loss) + + test_program = fluid.Program() + test_startup = fluid.Program() + with fluid.program_guard(test_program, test_startup): + with fluid.unique_name.guard(): + feed_order, loss, predict, accuracy, num_seqs = \ + create_model(args, num_labels, \ + is_prediction=True) + test_program = test_program.clone(for_test=True) + + exe = Executor(place) + exe.run(train_startup) + exe.run(test_startup) + + exec_strategy = fluid.ExecutionStrategy() + exec_strategy.num_threads = dev_count + + train_exe = fluid.ParallelExecutor( + use_cuda=args.use_cuda, + loss_name=loss.name, + exec_strategy=exec_strategy, + main_program=train_program) + + + test_exe = fluid.ParallelExecutor( + use_cuda=args.use_cuda, + main_program=test_program, + share_vars_from=train_exe) + + feed_list = [ + train_program.global_block().var(var_name) for var_name in feed_order + ] + feeder = fluid.DataFeeder(feed_list, place) + + time_begin = time.time() + total_cost, total_acc, total_num_seqs = [], [], [] + for batch_id, data in enumerate(train_data_generator()): + fetch_outs = train_exe.run( + feed=feeder.feed(data), + fetch_list=[loss.name, accuracy.name, num_seqs.name]) + avg_loss = fetch_outs[0] + avg_acc = fetch_outs[1] + cur_num_seqs = fetch_outs[2] + total_cost.extend(avg_loss * cur_num_seqs) + total_acc.extend(avg_acc * cur_num_seqs) + total_num_seqs.extend(cur_num_seqs) + if batch_id % args.skip_steps == 0: + time_end = time.time() + used_time = time_end - time_begin + current_example, current_epoch = processor.get_train_progress() + print("epoch: %d, progress: %d/%d, step: %d, ave loss: %f, " + "ave acc: %f, speed: %f steps/s" % + (current_epoch, current_example, num_train_examples, + batch_id, np.sum(total_cost) / np.sum(total_num_seqs), + np.sum(total_acc) / np.sum(total_num_seqs), + args.skip_steps / used_time)) + time_begin = time.time() + total_cost, total_acc, total_num_seqs = [], [], [] + + if batch_id % args.validation_steps == 0: + total_dev_cost, total_dev_acc, total_dev_num_seqs = [], [], [] + for dev_id, dev_data in enumerate(dev_data_generator()): + fetch_outs = test_exe.run( + feed=feeder.feed(dev_data), + fetch_list=[loss.name, accuracy.name, num_seqs.name]) + avg_dev_loss = fetch_outs[0] + avg_dev_acc = fetch_outs[1] + cur_dev_num_seqs = fetch_outs[2] + total_dev_cost.extend(avg_dev_loss * cur_dev_num_seqs) + total_dev_acc.extend(avg_dev_acc * cur_dev_num_seqs) + total_dev_num_seqs.extend(cur_dev_num_seqs) + print("valid eval: ave loss: %f, ave acc: %f" % + (np.sum(total_dev_cost) / np.sum(total_dev_num_seqs), + np.sum(total_dev_acc) / np.sum(total_dev_num_seqs))) + total_dev_cost, total_dev_acc, total_dev_num_seqs = [], [], [] + + if batch_id % args.save_steps == 0: + model_path = os.path.join(args.checkpoints, str(batch_id)) + if not os.path.isdir(model_path): + os.makedirs(model_path) + fluid.io.save_persistables( + executor=exe, + dirname=model_path, + main_program=train_program) + + +if __name__ == '__main__': + args = base_parser() + print_arguments(args) + main(args) -- GitLab