diff --git a/PaddleNLP/unarchived/chinese_ner/.run_ce.sh b/PaddleNLP/unarchived/chinese_ner/.run_ce.sh deleted file mode 100755 index 79b2da72c165a9b60b14f2748c8d46eb5b061a7d..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/.run_ce.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -export MKL_NUM_THREADS=1 -export OMP_NUM_THREADS=1 - - -cudaid=${chinese_ner:=0} # use 0-th card as default -export CUDA_VISIBLE_DEVICES=$cudaid - -FLAGS_benchmark=true python train.py --num_passes 300 --device GPU --enable_ce | python _ce.py - -cudaid=${chinese_ner_4:=0,1,2,3} # use 0-th card as default -export CUDA_VISIBLE_DEVICES=$cudaid - -FLAGS_benchmark=true python train.py --num_passes 300 --device GPU --parallel True --enable_ce | python _ce.py - -export CPU_NUM=1 -export NUM_THREADS=1 - -FLAGS_benchmark=true python train.py --num_passes 300 --device CPU --enable_ce | python _ce.py diff --git a/PaddleNLP/unarchived/chinese_ner/README.md b/PaddleNLP/unarchived/chinese_ner/README.md deleted file mode 100644 index a458c83b5f1ad9c007d35ddfb7a6578fb14bbf2a..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# 使用ParallelExecutor的中文命名实体识别示例 - -以下是本例的简要目录结构及说明： - -```text -. -├── data # 存储运行本例所依赖的数据，从外部获取 -├── reader.py # 数据读取接口, 从外部获取 -├── README.md # 文档 -├── train.py # 训练脚本 -├── infer.py # 预测脚本 -``` - -## 数据 -在data目录下，有两个文件夹，train_files中保存的是训练数据，test_files中保存的是测试数据，作为示例，在目录下我们各放置了两个文件，实际训练时，根据自己的实际需要将数据放置在对应目录，并根据数据格式，修改reader.py中的数据读取函数。 - -## 训练 - -通过运行 - -``` -python train.py --help -``` - -来获取命令行参数的帮助，设置正确的数据路径等参数后，运行`train.py`开始训练。 - -训练记录形如 -```txt -pass_id:0, time_cost:4.92960214615s -[Train] precision:0.000862136531076, recall:0.0059880239521, f1:0.00150726226363 -[Test] precision:0.000796178343949, recall:0.00335758254057, f1:0.00128713933283 -pass_id:1, time_cost:0.715255975723s -[Train] precision:0.00474094141551, recall:0.00762112139358, f1:0.00584551148225 -[Test] precision:0.0228873239437, recall:0.00727476217124, f1:0.0110403397028 -pass_id:2, time_cost:0.740842103958s -[Train] precision:0.0120967741935, recall:0.00163309744148, f1:0.00287769784173 -[Test] precision:0, recall:0.0, f1:0 -``` - -## 预测 -类似于训练过程，预测时指定需要测试模型的路径、测试数据、预测标记文件的路径，运行`infer.py`开始预测。 - -预测结果如下 -```txt -152804 O O -130048 O O -38862 10-B O -784 O O -1540 O O -4145 O O -2255 O O -0 O O -1279 O O -7793 O O -373 O O -1621 O O -815 O O -2 O O -247 24-B O -401 24-I O -``` -输出分为三列，以"\t"分割，第一列是输入的词语的序号，第二列是标准结果，第三列为标记结果。多条输入序列之间以空行分隔。 diff --git a/PaddleNLP/unarchived/chinese_ner/_ce.py b/PaddleNLP/unarchived/chinese_ner/_ce.py deleted file mode 100644 index a53d6fe5be564044a00c98bdcfeaec7f6884822e..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/_ce.py +++ /dev/null @@ -1,68 +0,0 @@ -# this file is only used for continuous evaluation test! - -import os -import sys -sys.path.append(os.environ['ceroot']) -from kpi import CostKpi -from kpi import DurationKpi -from kpi import AccKpi - -each_pass_duration_cpu1_thread1_kpi = DurationKpi( - 'each_pass_duration_cpu1_thread1', 0.08, 0, actived=True) -train_recall_cpu1_thread1_kpi = AccKpi('train_recall_cpu1_thread1', 0.08, 0) -each_pass_duration_gpu1_kpi = DurationKpi( - 'each_pass_duration_gpu1', 0.08, 0, actived=True) -train_recall_gpu1_kpi = AccKpi('train_recall_gpu1', 0.08, 0) -each_pass_duration_gpu4_kpi = DurationKpi( - 'each_pass_duration_gpu4', 0.08, 0, actived=True) -train_recall_gpu4_kpi = AccKpi('train_recall_gpu4', 0.08, 0) - -tracking_kpis = [ - each_pass_duration_cpu1_thread1_kpi, - train_recall_cpu1_thread1_kpi, - each_pass_duration_gpu1_kpi, - train_recall_gpu1_kpi, - each_pass_duration_gpu4_kpi, - train_recall_gpu4_kpi, -] - - -def parse_log(log): - ''' - This method should be implemented by model developers. - - The suggestion: - - each line in the log should be key, value, for example: - - " - train_cost\t1.0 - test_cost\t1.0 - train_cost\t1.0 - train_cost\t1.0 - train_acc\t1.2 - " - ''' - for line in log.split('\n'): - fs = line.strip().split('\t') - print(fs) - if len(fs) == 3 and fs[0] == 'kpis': - kpi_name = fs[1] - kpi_value = float(fs[2]) - yield kpi_name, kpi_value - - -def log_to_ce(log): - kpi_tracker = {} - for kpi in tracking_kpis: - kpi_tracker[kpi.name] = kpi - - for (kpi_name, kpi_value) in parse_log(log): - print(kpi_name, kpi_value) - kpi_tracker[kpi_name].add_record(kpi_value) - kpi_tracker[kpi_name].persist() - - -if __name__ == '__main__': - log = sys.stdin.read() - log_to_ce(log) diff --git a/PaddleNLP/unarchived/chinese_ner/data/label_dict b/PaddleNLP/unarchived/chinese_ner/data/label_dict deleted file mode 100644 index 7dfbc7c512a0c849c915ba78a367fe3cf547c351..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/data/label_dict +++ /dev/null @@ -1,49 +0,0 @@ -24-B -24-I -27-B -27-I -20-B -20-I -21-B -21-I -22-B -22-I -23-B -23-I -28-B -28-I -29-B -29-I -12-B -12-I -11-B -11-I -10-B -10-I -13-B -13-I -38-B -38-I -14-B -14-I -16-B -16-I -33-B -33-I -18-B -18-I -31-B -31-I -30-B -30-I -37-B -37-I -36-B -36-I -35-B -35-I -19-B -19-I -32-B -32-I -O diff --git a/PaddleNLP/unarchived/chinese_ner/data/test_files/test_part_1 b/PaddleNLP/unarchived/chinese_ner/data/test_files/test_part_1 deleted file mode 100644 index 233b942ffaa2a0442f4690203eca69798e17c40f..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/data/test_files/test_part_1 +++ /dev/null @@ -1,500 +0,0 @@ -;18528 45786 10718 1 348423 18528 45786 10718 408 2 1184 26 348423 0 370732 0 408 2 1184 0 13405 99 558667 2 1670 2 52 0 2 0 111;31 32 32 0 0 31 32 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;39 40 40 0 0 39 40 40 33 34 33 0 0 0 0 0 33 34 33 0 21 22 0 0 33 0 0 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;88 19554 67 31745 423 561 6463 0 49680 0 537 0 224317 0 13851 0 88778 0 23 1256 2013 46 380362 4832 16141 25 116 0 2 0 88 2384 110 28 18 0 2 0 1774 884 224 23 13 199 712 729 1256;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 0 0 0 1 2 2 0 0 0 0 0 0;13 33 0 33 33 34 33 0 33 0 0 0 21 0 33 0 0 0 0 33 33 0 0 21 22 0 33 0 0 0 31 32 32 32 32 0 0 0 1 2 2 0 33 34 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 379 2031 2675 691 14657 4817 1190 4223 108 643 62 6287 66 1311 406 58 2 92 401 257;0 0 0 39 40 40 40 0 0 0 0 0 0 0 0 0 0 0 17 18 18;0 0 33 47 48 48 48 33 33 34 34 0 33 0 33 33 0 0 31 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2397 7008 10 219 359 423 3 17 1 4 9;43 44 0 0 0 0 0 0 0 1 2;51 52 0 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;300 2 2077 2 4778 1 405 1 35 19 31 95;0 0 0 0 0 0 0 0 0 0 0 0;33 0 21 0 33 0 1 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1; -;277 2 80497 8869 104 2 322 5;0 0 0 0 0 0 1 2;0 0 33 33 33 0 1 2;1 1 1 1 1 1 1 1; -;625964 10 20 156 1 625964 31 273 1 140 1 1809 1 1526 1 2139 1 704 731 1538;0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0 0 5 6 6;0 33 34 33 0 0 29 30 0 33 0 33 0 33 0 33 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 49516 16 0 212 574 6251 1742 904 3 7779 492 1 3372 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 33 34 0 1 2;0 33 0 0 0 0 33 34 0 0 33 0 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2875 2267 1172 2151 3 39 13 324 242 46 605 510941 784 16181 139 154 2120 3644 0 7920 0 9255 0 6807 55 144 7 679 1 1115 40;0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 5 0 0 0 0 0 0 0;33 33 33 33 0 33 34 34 0 0 1 0 33 34 0 0 0 33 0 33 0 1 0 33 0 0 0 33 0 9 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3784 563 3648 1025 46 22417 6272 539 90 64 856 422 949 165 82 3784 1449 83 0;0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 33 34 0 13 33 0 0 0 33 0 33 0 0 33 34 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;490 73 2288 381 28116 5577 1136 1311 406 5;0 9 10 0 0 0 0 0 0 0;0 13 14 33 0 33 34 33 33 34;1 1 1 1 1 1 1 1 1 1; -;328 312 2017 445 437 0 2961 5807 2 210 2 35 19 2 41 72;0 0 0 39 40 0 0 0 0 0 0 0 0 0 1 2;13 0 0 47 48 0 35 36 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;25530 0 7437 0 2 0 2448 426 0 2 0 387 19;0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 31 0 0 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;48654 148 1295 101 37 91 180 1 4 9;9 10 0 0 0 0 0 0 1 2;13 14 33 34 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;88 27 13026 75 13026 376 67668 376 503 5031 1056 58;9 10 9 10 9 10 9 10 10 0 0 0;13 14 13 14 13 14 13 14 14 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1; -;34896 9148 1 34896 9148 37 1 410 34896 9148 104 26 182 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;33 33 0 33 33 33 0 33 33 33 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;54670 1105 1243 5100 1798 6085 252 110 10200 9467;17 18 18 18 0 0 0 0 0 0;13 33 34 33 0 33 0 21 22 33;1 1 1 1 1 1 1 1 1 1; -;35301 7453 14085 1479 3 3310 1 4 9;21 0 9 10 0 0 0 1 2;27 33 13 14 0 33 0 1 2;1 1 1 1 1 1 1 1 1; -;241 1355 178 733 305 91 180;0 0 0 0 0 0 0;35 36 36 33 33 0 0;1 1 1 1 1 1 1; -;4415 18090 497 2789 4919 34872 351 86208 2400 17406 2143 4239 922 10 16504 922 24 1 4 9;0 17 18 18 18 18 18 18 18 18 18 18 18 0 0 0 0 0 1 2;31 32 32 32 32 32 32 32 32 32 32 32 32 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;77781 726 2295 1554 4254 12944 53031 8582 125 58 2 34 41483;21 22 0 0 0 0 0 0 0 0 0 9 0;27 28 33 33 33 33 21 21 33 0 0 13 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;69685 251 545770 10 20 156 0 49 6272 2162 48 329 2527 2 2527 329 1538 1 69685 251 545770 3 6272 1576 32 5295 1 69685 251 545770 3 273 1 69685 251 545770 15 817;21 22 22 0 0 0 0 5 6 6 6 0 0 0 0 0 0 0 21 22 22 0 0 0 0 0 0 21 22 22 0 0 0 21 22 22 0 0;13 0 0 33 34 33 0 0 9 10 0 0 0 0 35 36 36 0 13 0 0 0 33 33 0 33 0 13 0 0 0 33 0 13 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49002 1484 939 1042 2889 1407 2 27064 1407 2 8293 5 0;0 27 28 0 0 0 0 0 0 0 1 2 0;35 33 34 33 33 33 0 21 33 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;139 29096 89981 11555 144 48398 14819 2 12155 71830 2407 48339 20107 4730 353 332 1 29096 89981 11555 633 71 2 340 247 2578 5;0 0 0 0 0 0 0 0 0 9 10 0 0 0 0 0 0 0 0 0 0 0 0 9 1 2 2;0 31 0 0 0 33 33 0 33 13 14 33 33 0 0 0 0 31 0 0 33 34 0 13 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;489 169 2298 275 6811 2737 329 1 34 1803 5;0 0 0 0 0 0 0 0 1 2 2;33 34 34 34 34 34 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1; -;8395 1531 46 0 52640 0 962 0 5918 0 2699 1457 86 0 2 0 5758 0 67 142 0 2 0 5758 62 34 66;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 9 0;33 33 0 0 29 0 0 0 33 0 33 33 34 0 0 0 31 0 0 33 0 0 0 31 0 13 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;85 97 1705 181 787 118 3280 2917 136 27 5483 3391 1 934 4053 5;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 34 34 34 0 0 0 0 0 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34 3156 4527 18117;9 0 0 0;13 33 39 33;1 1 1 1; -;20 10 38123 60 1 4 9;0 0 0 0 0 1 2;0 0 33 0 0 1 2;1 1 1 1 1 1 1; -;49 3337 894 48 2 141 2 35 887 55 2 31 95 2 4121 1047 19;19 20 20 20 0 0 0 0 0 0 0 19 20 0 0 0 0;0 25 26 0 0 33 0 33 33 34 0 33 34 0 1 2 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 3 158 164 3325 4306 3877 1 4 9;0 0 0 0 0 0 0 0 1 2;21 22 22 22 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;857 251 19508 0 103 6828 376 1342 5;9 0 0 0 9 10 10 1 2;13 0 21 0 13 14 14 1 2;1 1 1 1 1 1 1 1 1; -;14 15862 2785 6919 16 44 170 40 1 15862 2785 6919 926 1 15862 2785 6919 838 2 916 2093 163 7 702 2759 7 65;0 11 12 0 0 0 0 0 0 11 12 0 0 0 11 12 0 0 0 1 2 0 0 0 0 0 0;0 15 16 33 0 33 33 33 0 15 16 33 33 0 15 16 33 33 0 1 2 33 0 1 31 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4400 14052 85 869 367 6 4400 14052 85 724 534 40 6 14452 1528 3994 438 224 31 367 6 17415 25468;27 28 0 0 0 0 0 0 0 0 0 0 0 21 22 22 0 0 0 0 0 0 0;21 22 0 33 33 0 21 22 0 33 33 33 0 27 28 28 33 34 33 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1434 215 422 33 60 0 7445 91 8 22 10294 1434 1556 4495 3 4178 8 236 220 3 8 1887 677 43 94 10 162 38 298 227 3 8 12 1406 2640 5092 10 91 1 4 9;0 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 0 0 0 0 0 0 0 0 31 33 0 33 0 33 0 0 0 0 0 0 0 0 33 0 33 34 0 0 0 0 33 34 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5147 9423 75 2340 5;9 10 10 1 2;13 14 14 1 2;1 1 1 1 1; -;1795 1 6526 1 86267 3 655 0 2 0 1115;0 0 0 0 0 0 0 0 0 0 5;0 0 33 0 21 0 33 0 0 0 9;1 1 1 1 1 1 1 1 1 1 1; -;57148 1956 28475 2233 7104 0 2400 223 40028 6454 23 68 25 1 188 167;21 0 21 0 0 0 0 0 0 0 0 0 0 0 17 0;27 33 27 33 34 0 33 34 33 33 0 0 0 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;41 341 5044 13 145 8 1082 41 341 2444 13 145 1 1739 2325 1 35259;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 33 34 0 33 0 0 35 36 36 0 33 33 0 13;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;326 4996 3975 21 8 250 1082 391 3066 8 8 94 99 3111 3 10 331 326 8 1053 348 1860 254 8 165 921 5342 254 524 8 541 24121 27586 8 3112 1 4 9;5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;9 33 33 34 0 0 33 0 33 0 0 33 34 33 0 0 0 9 0 33 0 0 0 0 0 33 33 0 0 0 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6604 22 275 39 13 417 14135 56 3 5484 30975 8675 22014 10 62 0 66 408 388 37688 22014 642 388 1448 27290 2780 22014 715 388 515 22014 1104 388 63063 22014 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 10 0 0 0 0 0 1 2 2;33 21 22 33 34 0 33 0 0 33 33 35 36 0 0 0 0 0 0 33 34 0 0 21 22 22 22 0 0 13 14 0 0 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;48046 14942 2400 668 2898 38 0 56 0 5591 275 3451 1 233 922 14143 1 114 204;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;27 33 0 0 33 34 0 0 0 0 33 34 0 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;497606 7 19198 55703 7 65 3 339 196 1;0 0 0 0 0 0 0 0 0 0;0 0 0 0 33 34 0 33 33 0;1 1 1 1 1 1 1 1 1 1; -;395189 20 156 1 4 9;0 0 0 0 1 2;0 0 33 0 1 2;1 1 1 1 1 1; -;60226 946 2315 5404 62 165 3 1277 6759 93 1854 9545 66 1 4 9;17 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;31 33 34 33 0 0 0 33 33 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1433 1114 1180 19 7653 1828 23169 1 4 9;17 0 0 0 0 0 0 0 1 2;31 33 34 33 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1; -;67 22 8255 0 23 216 13 206 25 1 9302 201 1 6945;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 0 0 0 33 34 0 0 33 34 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11443 67218 63 18 1975 1 69 255 1 18 69 1 4958 24120 616 28 18 0;0 21 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 0;33 27 0 33 33 0 21 22 0 33 33 0 31 32 32 32 32 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;605565 141 534 1 605565 137 141 1 605565 2190 254 3 141 438 14 4675 141 201 16;0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 0;21 33 33 0 21 33 34 0 21 0 0 0 33 34 0 0 9 10 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15017 1365 30410 1339 42 12133 0 2191 117 407 34 42 23 35 1488 25 0 2 0 69 0 2 0 207 31;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 33 33 33 0 33 0 33 0 0 33 34 0 33 0 0 0 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;835 1506 915 10 9958 3 203 17 3422 714 1550 179 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 33 0 33 0 0 0 33 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;106 7138 14530 848 9034 8732 1483 998 2016 362 0 2 0 102 38 378 97 768 112 0 2 0 768 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 22 0 0 33 0 0 33 34 0 0 0 33 34 27 28 1 2 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3498 2 111;0 0 17;33 0 31;1 1 1; -;192 159 257 899 20589 1863 986 1341 3364 210 58 1 598 1754 257 4223 785 244;17 18 18 0 0 0 0 0 0 0 0 0 17 18 18 0 0 0;31 32 32 33 34 21 22 22 22 22 22 0 31 32 32 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 127 304 1121 2 315 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;222 13 470 6416 32 21663 13 332 108910 3 3842 1 4 9;0 0 0 21 0 21 0 0 0 0 0 0 1 2;21 22 0 27 0 0 33 34 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;139 17629 3571 22614 144 0 4556 1135 39870 8 6188 82 9099 8132 83 0 23 2645 9152 5 25;0 0 0 0 0 0 21 22 22 0 0 0 0 0 0 0 0 1 2 2 0;0 33 33 33 0 0 27 28 28 0 33 0 33 33 0 0 0 1 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10237 1833 46 15841 3 36165 10 13 523 19040 2 45 81 5;0 0 0 17 0 0 0 0 0 0 0 1 2 2;33 34 0 31 0 0 0 33 34 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4643 98 647 174 1222 1451 30 1 4 9;17 18 0 0 0 0 0 0 1 2;31 32 33 34 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1; -;39 57 414 1243 1152 244 1267 407 161 14716 90855 82 488 33313 83 2 2 1549 2 2 169 5 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;0 0 33 34 33 0 33 0 33 33 33 0 0 33 0 0 0 33 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;58754 2912 2 8946 18686 7073 898 10442 0 529 524 3903 14457 1749 118 17980 12014 2 1433 401;17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;31 33 0 33 27 0 0 33 0 0 0 33 0 0 0 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16354 873 26965 1215 755 798 3 2007 15 3424 1 4 9;19 20 20 0 0 0 0 0 0 0 0 1 2;25 26 26 33 33 34 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 6624 6887 725 175 404 81 5777 516 269 14736 0 129 42 2019 2 342 5 0;9 9 9 10 10 0 0 0 0 0 0 0 0 0 0 0 1 2 0;13 13 13 14 14 0 0 33 33 34 33 0 0 0 33 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;30152 755 44824 5359 6831 3 4304 178 33 2718 8 129 2 220 2671 84 2 260 6184 2 6044 1285 2 330 1715 2825;0 0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 0 33 33 33 0 33 0 29 30 0 0 0 0 0 0 0 33 34 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6791 8593 611 86 1 504 1 4 61;0 23 24 24 0 0 0 1 2;33 29 30 30 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;160685 2837 23 69600 178 25 46 82 63 21 340 8999 96 160 61968 7818 3 4851 2837 32 1065 3 8 1257 9543 0 26 490 214 115 10 6538 440 160 3 178 8 165 10 30 7818 302 11717 8 1257 9543 0 26 490 860 7 7 7 83;11 12 0 9 0 0 0 0 0 0 9 0 0 0 0 0 0 11 12 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;15 16 0 13 0 0 0 0 33 34 13 33 0 0 0 33 0 15 16 0 33 0 0 0 33 0 0 0 0 0 0 33 33 0 0 0 0 33 34 0 33 0 33 0 0 33 0 0 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2700 33587 632 825 18653 102 137 116 71 1 123 1 147 0 2 0 489 2182 5;17 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 1 2 2;13 11 39 40 33 33 21 33 34 0 33 0 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1321 3704 55843 27309 12641 8 77 804 91 180 59 59 1 3704 55843 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 27 29 33 0 33 34 0 0 0 0 0 0 27 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;46740 2337 41307 5432 477 85 2 7642 1 4 61;0 0 0 0 0 0 0 0 0 1 2;0 33 21 22 22 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;179 21233 5465 825 67 804 21233 5465 825 0 80 160 6431 17 0 2 0 11 76 0 2 0 4904 0 2 0 2240 0 2696 0 709 197;31 32 32 32 0 0 31 32 32 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 1 2;39 40 40 40 0 0 39 40 40 0 0 0 33 0 0 0 0 33 34 0 0 0 31 0 0 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;617 7496 3358 1182 8 190 3 4260 7 7 7 0 2 0 649 848 5;9 19 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;13 25 21 22 0 0 0 33 0 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 117 7 1876 16 139 95407 3709 34357 8106 6368 3709 2954 3709 112190 154251 11373 1296 144 73 2294 1968 5354 12580 56 1 27477 723 384 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 0 17 18 18 0 0 1 2;0 0 0 33 0 0 27 0 21 35 36 0 0 0 33 37 33 34 0 0 0 33 33 33 34 0 31 32 32 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;406 1083 96 136 3 3223 3029 8553 8 236 406 1083 8 440 406 5804 0 2 0 612 295 42;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 9 0 0 0 33 33 33 0 0 33 9 0 33 33 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;402576 2 24418;0 0 0;33 0 0;1 1 1; -;59982 11 64 24 17 305 3910 24 17 19101 279 184 4404 93 24 17 1 4209 872 43 1 4 47;17 0 0 0 0 0 0 0 0 0 5 6 6 0 0 0 0 17 0 0 0 1 2;31 33 34 0 0 33 33 0 0 0 9 10 10 33 34 0 0 31 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;37695 73 6097 3349 1568 118 4830 13 8932 36523 10566 1364 3692 8985 8 3946 10815 8 31865 333 8 1 4 209 132;37 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;45 0 0 33 34 0 0 0 33 33 0 0 21 22 0 33 33 0 33 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1985 7932 900 470 1758 8 3029 30 2616 1961 1 4 9;0 0 0 0 0 0 0 0 0 0 0 1 2;33 21 0 0 0 0 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 235 16512 1247 48 41 20955 1 14 315 301 36 4128 301 16 1 3820 1635 258;0 0 0 0 0 0 0 0 0 0 0 0 27 28 0 0 0 0 0;0 33 33 34 0 0 33 0 0 33 34 0 35 36 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 564 407 1305 1986 42136 63 178 1506 446 16 1 564 407 1305 1986 42136 63 178 1506 1 4 493;0 9 0 0 0 11 0 0 0 0 0 0 9 0 0 0 11 0 0 0 0 1 2;0 13 33 34 33 15 21 22 0 33 0 0 13 33 34 33 15 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;233 813 58330 60223 19924 1260 1 314 159 142 5;0 0 0 0 0 0 0 1 2 2 2;21 22 22 33 34 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1; -;14 2856 3 4463 16 2856 3 4463 37 1 2856 3 4463 104 1 2856 3 4463 87 0 2 427 262 318 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 0 33 0 33 33 0 33 0 33 33 0 33 0 33 31 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1903 6109 33932 28191 8745 45246 1245 3 3101 657 7 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 0 47 48 47 48 48 0 33 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;23888 51 276873 10755 10241 10221 2984 0 81 10221 2524 58600 1 328 110 1 328 5;17 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;31 0 27 33 33 33 33 0 0 33 33 33 0 13 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;17770 39 151 224 19 31 95 0 1 12766 836 264 201;0 0 0 0 1 2 0 0 0 0 0 0 0;33 0 33 0 1 2 33 0 0 0 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;500 1296136 19548 1177 87946 847 8210 5404 778 19 360 38 938 2 315 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 33 33 33 33 33 0 33 33 34 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 6050 33 11 13 585 171 21 8 11 13 29 278 0 11 145 11280 15 70 22191 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 0 33 34 33 0 0 0 33 34 0 0 33 34 33 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;181 1279 1312 450 4764 129 1813 53824 76 1250 17923 71927 0 181 8368 7849 2 45 81 5;0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;21 22 33 0 0 0 0 33 34 0 33 33 0 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 5427 7159 2148 14375 2 1587 25203 16 37 6 87 6 44 6 4125 6 598 27 30625 19168 495 1567 4123 244 2 395 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 0 17 18 18 18 18 18 18 18 0 1 2;0 33 33 0 31 0 33 33 0 33 0 31 0 33 0 29 0 13 14 33 31 21 22 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7834 41799 39 1267 224 2 2 35 979 19 31 291 2 205 354 2 41 72;0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 1 2;33 34 0 33 0 0 0 33 33 1 2 33 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;275 5 11018 27 4824 18 46 106 82 666 83 8593 2109 18060 1986 82 4933 83 1 137 2321 1 588 5;17 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 13 0 33 34 0 0 0 0 0 33 33 33 33 0 33 0 0 21 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15 3489 1729 56 152 13584 4811 118 2262 103 3 7503 1 4 209 132;0 0 0 0 0 29 0 0 0 0 0 37 0 17 23 24;0 33 0 0 0 37 0 0 33 34 0 33 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;380187 0 869278 790 0 2 0 5523;0 0 0 0 0 0 0 0;0 0 0 0 0 0 0 33;1 1 1 1 1 1 1 1; -;22 1244 5519 3102 129 8 7293 3767 1 3283 3102 129 43 1 4 47;0 0 0 23 24 0 0 0 0 0 0 0 0 0 1 2;0 33 33 29 30 0 33 33 0 21 22 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;794 691 554 521 10170 295 4876 1379 220 1 4 61;0 0 0 0 0 0 0 0 0 0 1 2;25 0 33 34 33 21 22 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;49 51604 133522 48 27805 133522 2454 44395 72899 1 405 1 35 19 31 95;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 27 0 27 27 33 33 0 0 1 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;779 2816 5515 853 176 15940 1 4 61;0 0 0 0 0 0 0 1 2;21 22 33 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1; -;546 1989 51 90 699 215 166 7338 1 357 546 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 33 34 0 0 33 0 21 22 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;546 54 115 1244 11 2201 638 21 1 6927 14618 43 1 4 47;1 2 2 0 0 0 0 0 0 0 19 0 0 1 2;1 2 2 33 0 33 33 0 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;142 7700 383 1815 1095 2 10482 355 2 0 39 57 535;0 0 0 0 0 0 0 0 0 0 0 0 0;33 47 21 22 33 0 33 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;30798 148 2183 4891 376 7122 2719 5992;9 10 9 10 10 0 0 0;13 14 13 14 14 21 22 33;1 1 1 1 1 1 1 1; -;56 8257 13 1174 4629 21 8 348 12 1117 8 4786 54 298 30 3199 11546 254 11325 1608 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 0 17 23 24;0 33 0 33 0 0 0 0 0 0 0 0 33 34 0 0 0 0 29 30 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;404 854 3316 129 22915 282 1 404 854 3316 19 1 5502 315;45 46 46 46 21 0 0 45 46 46 0 0 45 46;53 54 54 54 27 33 0 53 54 54 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7763 1661 249 90 8913 49 11479 1171 48 672 38 3 3665 1251 1 76 1922 1 5841 523;0 0 0 0 0 45 46 46 46 0 0 0 0 0 0 0 0 0 0 0;39 0 33 34 33 33 34 34 34 33 0 0 33 0 0 53 54 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;37215 2563 3597 11 122 12 3819 51 1722 33 3 134 690 10188 1 1394 2246 1 1539 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 21 22 0 0 33 34 0 33 33 34 0 0 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9838 4416 30 6636 1 5194 3026 638 1 5194 3026 2786 1630;19 20 20 20 0 0 0 0 0 0 0 0 0;33 34 34 34 0 1 2 33 0 1 2 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;218 10572 237 28377 34 172 1265 904 1 188 69 108;17 18 18 0 0 0 0 0 0 5 6 0;31 32 32 33 21 22 22 22 0 9 10 33;1 1 1 1 1 1 1 1 1 1 1 1; -;2 0 389 18130 148 339 27603 5233 359 0 2 0 693 51 26 26 13264 1682 7 5281 2 18745 7 65 0;0 0 17 18 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 13 13 14 33 33 39 0 0 0 0 25 26 26 26 33 33 0 21 0 39 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11254 12689 59 171631 117 50 25338 18961 53 31 291 53 387 5 8 19 35 31 95;0 0 0 21 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;33 33 0 27 0 0 33 33 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2245 570 284 4006 38 1009 6 1009 91 38 3152 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 33 34 0 0 0 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6002 14970 2438 74411 101 450 0 2569 15410 125 5521 3719 501 369 2 45 81 5;21 22 0 0 0 0 0 0 0 0 0 37 38 0 0 1 2 2;21 22 33 33 0 0 0 0 33 33 0 33 34 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;524 868 44444 5;1 2 2 2;21 22 22 22;1 1 1 1; -;593 1764 174 62 424 66;21 22 22 0 0 0;27 28 28 0 33 0;1 1 1 1 1 1; -;11280 3 2371 19197 1123 20 17 1 4 9;0 0 0 0 0 0 0 0 1 2;33 0 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;3866 8647 3492 177 3604 1472 86 3879 21 8 397 236 1335 13 964 8 12 9 227 1351 1801 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;51 11 9 0 33 33 34 33 34 0 33 0 0 21 22 0 33 34 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1873 2165 1155 1 1873 2165 1155 37 1 1873 2165 1155 104 26 182 0 2 0 111;23 24 31 0 23 24 31 0 0 23 24 31 0 0 0 0 0 0 17;29 30 39 0 29 30 39 33 0 29 30 39 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;896 67419 15870 832 58 2 119 2 35 19 2 41 72;17 17 0 0 0 0 0 0 0 0 0 1 2;31 31 33 21 22 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 89 668 782 15 143 17 1 4 9;0 0 0 0 0 0 0 0 1 2;0 33 34 0 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1; -;12423 5424 7204 23810 15158 8 1279 106276 2476 346 17135 0 566 0 3315 5;21 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;27 33 33 33 33 0 0 0 33 34 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;69 3121 3121 3121 0 102 2 378 2 1752 0 1240 1877 40862 17 1400 574 14495 46 221 620 302 38 298 59 3096 1877 1720 96 122 25499 17 0 2 0 69 3121 3121 3121 0 3622 28249 100 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 0;21 22 22 22 0 33 0 33 0 33 0 33 0 33 0 0 0 33 0 33 34 0 0 0 0 33 0 33 34 0 33 0 0 0 0 21 22 22 22 0 0 27 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;132 6377 51 22 1970 15620 5576 12952 373 16058 3488 1742 8 11 356 930 29 3643 614 477 1 4 209 132;0 0 0 0 0 37 38 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;33 33 0 0 0 45 46 0 0 0 33 34 0 33 34 33 34 33 33 34 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11824 30 172 45 3 2070 1 4 61;5 0 0 0 0 0 0 1 2;9 0 33 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1; -;48853 13 682 113817 3 185 1 185;0 0 0 0 0 1 0 1;33 34 34 33 0 1 0 1;1 1 1 1 1 1 1 1; -;192768 0 217587 0 29957 0 43385 0 338883 0 11140 0 239 0 27649 0 52966 0 7088 4474 3141 0 55876 0 2 0 10870;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 21 0 33 0 33 0 21 0 33 0 0 0 33 0 0 0 33 33 33 0 33 0 0 0 9;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 16032 66541 16 1 16032 66541 14 260 26 44 26 37 16 1 16032 66541 260 1197 201 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1;0 33 33 0 0 33 33 0 33 0 33 0 33 0 0 33 33 33 33 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5006 6618 5189 250 3626;9 37 0 0 0;13 33 33 0 33;1 1 1 1 1; -;18460 148 39 4959 691 82 10856 1063 83 3561 484 1122 1508 7 2210 40 2 1028 35 170 1229 2 3492 1196;9 10 0 0 0 0 0 0 0 0 0 39 40 0 0 0 0 0 0 0 0 0 0 0;13 14 0 33 0 0 33 0 0 0 33 47 48 21 22 33 0 33 33 33 33 0 9 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;575757 0 23 794 25 0 2 0 5523;0 0 0 0 0 0 0 0 0;0 0 0 25 0 0 0 0 33;1 1 1 1 1 1 1 1 1; -;60420 3 5216 1 9400 7231 43 1 4 47;21 0 0 0 0 0 0 0 1 2;33 0 33 0 27 28 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;22 12 10 13283 8 7800 98 647 8 96 15 640 1419 98 2119 2 247;0 0 0 0 0 17 18 0 0 0 0 0 17 18 0 0 17;55 56 56 21 0 31 32 33 0 21 22 33 31 32 21 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 15305 32638 48 0 2 0 14021 1177 31;0 0 0 0 0 0 0 0 0 0;35 36 36 36 0 0 0 13 33 33;1 1 1 1 1 1 1 1 1 1; -;126 2933 331 699 30 12022 17052 212 17052 160 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 0 0 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;338 1150 27570 7553 3227 0 3854 357 658 17511 10295 1 188 634;9 0 0 17 18 0 0 9 0 0 0 0 0 0;13 33 33 25 26 0 0 13 0 33 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;377 177 1141 265 57 6 1141 177 2572 265 154 6 2572 177 377 265 85 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 23 24 0 0 0 0 1 2;33 34 34 0 0 0 33 34 0 0 0 0 29 30 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;397 467 4495 183 93 594 2061 183 24 0 730 12 502 0 1373 0 742 742 742 59 1 4 9;0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 29 30 0 0 0 31 0 0 0 33 33 34 0 33 0 33 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6355 52964 3731 9403 1 4 61;0 39 0 0 0 1 2;33 47 33 33 0 1 2;1 1 1 1 1 1 1; -;966074 67 4577 173 1 4 9;21 0 0 0 0 0 0;27 0 27 28 0 1 2;1 1 1 1 1 1 1; -;3694 129 1485 331 795 17410 6 12 9 1351 156 1 4 9;23 24 0 0 0 0 0 0 0 0 0 0 1 2;29 30 33 0 33 33 0 33 34 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;81 2056 8636 559 28 18 6 81 2056 8636 559 28 18 128 277 6 403 6 843 123 6 147;17 18 18 18 18 18 0 17 18 18 18 18 18 0 0 0 0 0 0 0 0 0;0 0 0 33 33 34 0 0 0 0 33 33 34 1 2 0 33 0 21 22 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;820 45 12820 298 820 29 2843 951 10 15 441 1 4 209 132;0 0 0 0 0 0 11 12 0 0 0 0 17 23 24;0 35 36 0 0 0 15 16 0 0 33 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10203 518 5793 15 517 1300 455 2772 17 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 0 33 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;115950 1397 10837 18583 63 21 8 15752 33 63 1651 8 89 11 152 1070 84 0 2 0 1715 4347 0 2 0 9676 210 2 439 210 1285 197;0 9 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 9 0 0 0;0 13 33 31 33 34 0 33 0 0 0 0 33 34 0 0 0 0 0 0 33 34 0 0 0 1 2 0 13 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1649 101 82 446 83 5318 40074 110 118 269 8014 2 247 119;0 0 0 0 0 0 0 0 0 0 0 0 17 18;33 34 0 33 0 33 33 33 34 34 33 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 11525 7764 41 589 48 57 430 1587 224 231 0;19 20 20 20 20 20 0 0 0 0 0 0;0 25 26 26 26 0 0 0 33 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1; -;642478 0 2 0 1062 27 39 127 169 107 412 4518 98 3174 1062 39 127 107 4990 697;21 0 0 0 17 18 18 18 18 18 17 18 18 18 18 18 18 18 0 0;27 0 0 0 31 32 32 32 32 32 31 32 32 33 13 33 34 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;984 113 287 182 5;17 18 18 0 0;31 32 32 33 34;1 1 1 1 1; -;14 3632 2638 86 170 55 5318 40 16 3632 2638 86 695 55 622 57 7 85 40 1 1249 94 78 40 5;0 5 6 6 0 0 5 0 0 5 6 6 0 0 0 0 0 0 0 0 0 0 0 0 0;0 9 10 10 33 34 33 33 0 9 10 10 33 0 33 34 0 0 33 0 1 33 34 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2914 2134 16 44 170 40 1 2914 2134 926 1 2914 2134 838 2 916 2093 163 7 702 2759 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0;0 33 33 0 33 33 33 0 33 33 33 0 33 33 33 0 1 2 33 0 1 31 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1120 317 22 4843 0 39 812 224 0 2 0 19 31 95 0 2 0 1120 317 22 4843 0 2 0 3589 2196;0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 0 0 33 0 0 0 0 1 2 33 0 0 0 33 34 34 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;139331 2407 2374 1 2087 5;9 10 0 0 1 2;13 14 33 0 1 2;1 1 1 1 1 1; -;41382 40157 7844 10 250 15684 8251 3 60 1 487 3 2072 32 1541 1 114 204;21 22 0 0 0 0 9 0 0 0 0 0 17 18 0 0 1 2;27 28 33 0 0 33 13 0 0 0 33 34 31 32 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;64 20 6066 212 1677 8935 21900 2564 3 4555 5638 625 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 33 34 34 0 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6363 1362 257 3 257 986 1 4 9;17 18 18 0 0 0 0 1 2;31 32 32 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1; -;710 11219 15092 13762 10 9472 16550 181 8 7120 15 3072 60 719 15 767 60 0 2 2 0 391 3375 170 738 372;0 0 0 0 0 35 36 36 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;0 0 33 33 0 43 44 44 0 33 0 33 0 33 0 0 0 0 0 0 0 1 2 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 21467 2017 83 607485 46 67 218 1086 29555 3 6087 1 44 131 1 342 5 0;0 21 22 0 21 0 0 17 18 0 0 0 0 0 0 0 1 2 0;0 27 28 0 27 0 0 31 32 33 0 33 0 33 33 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 10476 1126 83 1472 86 1 4 61;0 0 0 0 0 0 0 1 2;0 0 0 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1; -;9672 8088 11 1451 1851 8 42 4594 9672 8088 118 3 1176 84 1 4 209 132;37 38 0 0 0 0 0 0 37 38 0 0 0 0 0 17 23 24;33 34 33 34 33 0 33 34 33 34 0 0 33 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 35431 16 109903 0 642 0 16385 7 0 87190 0 2 0 13933 0 6613 14 1288 16 53 2448 426 46 49 617 190 2338 3584 629 4198 48 53 31 291 53 387 5 8 19 35 31 95;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;0 33 0 21 22 22 0 33 0 0 27 0 0 0 21 22 22 0 33 0 0 0 0 0 0 13 33 34 33 33 33 0 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;130063 2 5613 832 5;0 0 1 2 2;33 0 1 2 2;1 1 1 1 1; -;44781 10 13 166 8266 8 5808 274 455 212 12039 4592 8 136 212 457 122 5696 27859 84;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 34 34 0 33 0 0 0 33 34 0 33 34 33 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;658 5476 152 6577 45 221 6 333 21062 4437 681 284 11547 2638 63 6550 23236 26 451 6 5476 586 2004 121 228 7 2835 8796 6 541 17047 14771 166 681 43429 14167 62 11537 681 2638 654 3 1913 121 228 7 85 892 6 2758 30 509;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 40 0 0 0 0 0 0 0 0 0 0 0;0 47 0 21 22 22 0 0 33 33 0 0 33 33 0 33 0 0 0 0 47 33 0 0 0 0 33 33 0 0 33 33 0 0 33 33 0 33 0 47 48 0 33 0 0 0 31 32 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3518 16241 167 1 3518 16241 167 261 0 2 2818 434 128 0 408 2074 25983 7 2171 2828;9 0 0 0 9 10 10 0 0 0 0 0 0 0 0 0 0 0 0 0;13 33 33 0 13 33 33 33 0 0 0 33 33 0 33 34 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2904 4062 8132 1340 57 169 1439 449 91 4548 1 4 9;0 0 0 0 0 23 24 0 0 0 0 1 2;53 54 54 54 0 29 30 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1137827 1 14 443 71 1 1489 71 1 908 71 1 434 71 1 123 147 1 116 71 16 196 2 1841 510;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;27 0 0 33 34 0 33 33 0 33 34 0 33 33 0 33 33 0 33 34 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;656 11247 310 2 315 2 35 19 2 41 72;9 0 0 0 0 0 0 0 0 1 2;13 21 22 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;509 431 386 3 73 798 341 21 13 422 44837 8 1091 1670 386 331 9370 59;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 21 22 0 33 34 33 34 33 34 33 0 0 33 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;653 12043 11138 6 5089 23 63854 25 1890 6 12043 1237 400 6 404 358 11138 6 11138 1237 2668 8 12043 27337 8 435 16531 6 11138 3037 1785 8 2907 8973 8 7265 0 2 0 1552 193085 3 1899 204;0 0 0 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 23 0 1 2;33 21 22 0 33 0 21 0 0 0 37 33 34 0 0 0 29 0 29 33 34 0 37 33 0 33 33 0 29 29 30 0 0 0 0 0 0 0 0 13 29 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;77332 1507 197 818 150 108;17 18 18 18 18 18;13 14 31 32 32 32;1 1 1 1 1 1; -;284 25534 63 3611 3 20418 1 34 586 69 5 0;0 0 0 0 0 0 0 1 2 2 2 0;0 33 0 33 0 33 0 1 2 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1; -;15799 2162 685 94 0 373 305 303 6098 17 2 34 186 5 2 34 1109 339 411 128 2 1308 325 6 1231 6 1118 6 2351 6 861 6 110 449 71;5 6 0 0 0 0 0 0 5 0 0 1 2 2 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;9 10 33 33 0 0 33 33 33 0 0 1 2 2 0 13 33 33 34 34 0 33 33 0 33 0 33 0 33 0 33 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 10092 2127 6908 682 446 16 1 10092 2127 6908 682 63 178 1506 1 4 493;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 33 34 33 0 0 0 0 33 34 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;105 9957 1507 280 282 1 9957 1507 2247 557 282 26 2312 3105 26 13 3675 14 6494 768 282 16;0 9 10 0 0 0 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 13 14 33 34 0 13 14 33 34 33 0 33 0 0 0 33 0 31 32 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 352 789 30274 87 57 2 24745 30274 37 826 16 37 1 87 1 44 0 2 427 262 318 5;0 0 9 31 0 0 0 0 31 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 13 39 31 0 0 0 39 33 34 0 33 0 31 0 33 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2350 27923 3 98 174 9443 325 602 294 1 1451 359616 1 504 1 4 61;0 0 0 0 0 0 0 0 0 0 21 22 0 0 0 1 2;33 33 0 33 34 33 33 33 33 0 27 28 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4090 4790 311 51465 0 48358 300 55 40 36 4090 4790 311 51465 0 48358 300 763 55 713 695 55 0 57 7 228 1 9330 40 199;45 46 46 5 0 0 0 0 0 0 45 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 9 0 27 33 34 33 0 33 34 34 9 0 27 33 33 0 33 33 0 0 0 0 0 0 9 10 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;598 27 3385 5925 192 119 28 18;17 18 18 18 18 18 18 18;13 14 0 0 33 34 33 34;1 1 1 1 1 1 1 1; -;11046 4097 57 1357 1 11046 4097 57 1357 37 1 410 11046 4097 57 1357 104 26 182 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;33 0 0 0 0 33 0 0 0 33 0 33 33 0 0 0 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 11664 36156 9005 16 1 11664 36156 9005 44 1 37 1 13 2514 5;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 33 53 33 0 0 33 53 9 10 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2397 5939 54976 580 13 29 2498 17935 91 180 1 4 9;17 18 18 0 0 0 0 0 0 0 0 1 2;31 32 11 0 33 34 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;28292 46 5795 5568 12562 0 5542 9424 12130 1 268 131 1 342 5 0;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;31 0 33 0 33 0 33 0 33 0 21 33 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34102 1087 19003 929 2752 81 756 2 2 34102 148 2 342 5 1684 131 0;21 0 0 0 0 0 0 0 0 9 10 0 1 2 9 0 0;21 33 33 33 34 0 33 0 0 13 14 0 1 2 13 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9029 254 122 2712 6509 2497 2758 9019 8 11 668 24 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 33 33 33 0 33 0 55 56 56 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;29191 1630 1574 161 28 18 2 277;17 18 18 18 18 18 0 0;31 32 32 32 32 32 0 0;1 1 1 1 1 1 1 1; -;14 50 16271 1258 16 1 50 16271 1258 14 260 26 44 26 37 16 1 50 16271 1258 260 1197 201 0 2 0 111;0 21 22 0 0 0 21 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1;0 27 28 0 0 0 27 28 0 0 33 0 33 0 33 0 0 27 28 0 33 33 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;140 1545 1372 1 4 61;0 0 0 0 1 2;33 33 0 0 1 2;1 1 1 1 1 1; -;140 7401 51 10486 600 1 4 9;0 0 0 0 0 0 1 2;33 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1; -;14 281 11601 711 79362 16 281 11601 711 79362 37 1 281 11601 711 79362 104 1 281 11601 711 79362 87 0 2 427 262 318 5;0 23 24 24 0 0 23 24 24 0 0 0 23 24 24 0 0 0 23 24 24 0 0 0 0 0 0 1 2;0 29 30 30 0 0 29 30 30 0 33 0 29 30 30 0 33 0 29 30 30 0 31 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2113 138 2602 1617 1 4 61;0 0 0 0 0 1 2;33 34 33 33 0 1 2;1 1 1 1 1 1 1; -;27493 3683 5390 2 2240 253 2 8505 687;0 0 0 0 0 0 0 1 2;33 34 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1; -;487 134 10 1136 0 9177 0 15936 0 59432 0 2501 0;0 0 0 0 0 0 0 0 0 33 0 0 0;0 33 34 34 0 53 54 54 0 25 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;10119 5950 39616 164 13998 11015 502 1 2178 10119 5950;0 0 0 0 0 0 0 0 0 0 0;33 34 33 34 34 33 0 0 1 33 34;1 1 1 1 1 1 1 1 1 1 1; -;3263 443;0 0;33 33;1 1; -;2350 23229 2472 3 1546 67 2223 435 2189 323 756 208 53 53 568 28719 186 1336 435 121 3438 2 2 49 1492 142 257 6172 48 102 38 2835 276 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 28 28 28 28 28 0 0 0 0 0;33 21 22 0 33 34 34 33 33 34 33 33 33 34 0 33 34 33 33 0 0 0 0 0 33 34 34 34 0 33 34 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2178 31 363 1 195 1621 40 5419 363 1 94 56 170 363 2178 73 78 2 6735 78 211;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0;21 22 22 0 0 0 33 33 0 0 33 0 33 0 1 2 2 0 21 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;88 27 71 345 120 108 102 38 2340 71 345 1530 2877 2431 10562 2 2340 50 243 211 176 243 120 150 791 833 481;17 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;31 32 32 32 32 32 33 34 33 34 33 34 33 33 33 0 33 33 34 33 0 33 34 33 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3330 3 5618 2330 3468 10 6365 5516 2 17616 145 1125 5 2 17616 145 2116 2 141 2332 2 141 40 2 163 7 4710 9852 7 65;0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0;33 0 33 33 0 0 33 33 0 0 0 1 2 0 0 0 33 0 1 2 0 33 34 0 33 0 45 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 1534 930 3 2544 390 4670 8 321 45 3 12 2046 2100 1 8984 100 1 110 54 115 100;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0;0 0 0 33 34 0 21 0 33 0 33 33 34 33 34 33 0 11 33 0 9 10 10 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;163604 0 357605 0 62538 0 66028 0 36 0 105404;0 0 0 0 0 0 0 0 0 0 0;0 0 11 0 0 0 25 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1; -;702 38 13 3918 38 1207 38 1552 3 626 1 4 9;0 0 0 0 0 0 0 9 0 29 0 1 2;33 34 0 33 34 33 0 13 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;489 169 2298 275 14026 329;0 0 0 0 0 0;33 34 34 34 34 34;1 1 1 1 1 1; -;1404 1881 8889 5725 0 550 8213 77 41 4390 10 2065 2 1404 2 3931 1988 2 1881 1988 2 10596 300 5 36 10596 3321;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;33 33 33 33 0 33 33 33 34 33 0 0 0 33 0 33 33 0 33 33 0 51 1 2 0 51 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34 169 98 329 257 0 3032 1321;17 18 18 18 18 0 0 0;31 32 32 32 32 0 33 34;1 1 1 1 1 1 1 1; -;70303 106 248 10396 1730 1540 23 1235 189 25 1 86388 3336 1 3644 55 7 679 1 1115 40;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;35 36 36 36 33 34 0 0 0 0 0 27 28 0 33 0 0 33 0 9 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;946 2200 25168 1282 9997 0 1957 24962 3 19107 1 1422 217 415 5 1 1572 7 65;0 37 38 38 0 0 0 0 0 0 0 1 2 2 2 0 0 0 0;33 45 46 46 0 0 0 33 0 33 0 1 2 2 2 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 520 8170 289 1803 1092 83 10 520 7897 2897 8089 0 2 0 3353 2601 0 2 0 3908 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 34 0 0 0 0 33 33 0 0 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4395 2472 3456 3 1847 202 1 4 9;0 0 0 0 0 0 0 1 2;33 34 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1; -;157 4903 16800 13112 19290 7871 138 1 157 16800 7871 138 44 1 157 16800 7871 138 87 1 157 27 1145 9105 458 138 28 18;9 9 9 9 9 0 0 0 0 9 0 0 0 0 9 10 0 0 0 0 9 10 0 0 0 0 0 0;13 14 33 33 13 33 34 0 13 14 33 34 33 0 13 14 33 34 31 0 13 14 33 33 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;187 149535 22 26301 3 418 4110 1 418 4110 747 1 6945;0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 34 0 33 34 0 33 34 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;157 890474 1107 28 18 2 277;17 18 18 18 18 0 0;13 27 33 33 34 0 0;1 1 1 1 1 1 1; -;187 74 10772 1632 4740 496 107 3862 176 1632 3556 1664 379 4573 41157 270 138 366;0 0 17 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0;0 0 13 14 33 25 26 33 0 0 33 34 0 33 33 34 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7327 107 3766 1245 5288 0 4977 2898 9742 777 453 4891 453 12158 1 4154 1693 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 34 33 34 33 0 33 0 33 33 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;193 40 2 26748 235 139 163 7 1977 47766 7 65 144;0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 0 33 33 0 33 0 33 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;13764 9669 23 68 25 2 105245 0 11756 0 38080 2 412 2 588 648 5;11 12 0 0 0 0 0 0 0 0 0 0 9 0 1 2 2;15 16 0 0 0 0 21 0 35 0 15 0 13 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 4425 86 37 16 4425 86 104 37 1 4425 86 44 0 2 0 34 352 259;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 21 22 33 0 21 22 33 34 0 21 22 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;20177 4703 23 88 25 1023 119 28 18 1 1256 712 1 3411 116;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;31 33 0 13 0 33 34 33 34 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;169 1439 207 323 17 6612 87958 8 421 10679 2758 240 59 2 1353 99;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 22 22 0 33 33 0 33 34 34 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2551 44 0 2551 301 0 4701 141 0 6252 141 0 11767 629 430;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 33 33 0 33 34 0 9 10 0 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2121 120 2632 637 1653 1689 13421 2727 3 1233 45242 1017 84 84 84 0 2 0 1412 3306 0 2 0 3385 1321 5 36 2632 42 100;0 0 9 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 1 2 2;0 33 13 13 14 0 0 33 0 0 33 0 0 0 0 0 0 0 33 34 0 0 0 1 2 2 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8709 20354 10 20 676 1 4 209 132;0 0 0 0 0 0 17 23 24;33 33 33 34 33 0 31 29 30;1 1 1 1 1 1 1 1 1; -;3759 1123 283 1098 4850 0 21405 407 1114 152 4560 346 2640 516 2 45 81 5;0 0 0 0 0 0 0 0 0 0 17 18 0 0 0 1 2 2;33 34 33 34 0 0 33 0 33 0 31 32 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1199 923 113 2807 168 1845 13 24077 168 468 355 1 4 61;0 9 10 0 0 39 40 0 0 0 0 0 1 2;33 13 14 33 33 47 48 33 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15536 72706 9429 122 21 8 1482 11 89 11 698 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 0 21 22 0 0 33 34 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 20632 15704 1818 830 178 6 20632 15704 1818 830 178 446 3639 37 14 20632 15704 1818 830 178 123 1413 6 147 6 826 2073 16 11055 73 5112 261;17 18 18 18 18 0 0 17 18 18 18 18 0 0 0 0 17 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0;13 14 14 15 16 0 0 13 14 15 16 0 33 33 33 0 13 14 15 16 0 33 33 0 33 0 33 34 0 33 34 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;19920 3358 649 141 49 7561 1952 8095 48 985 126611 280 0 2 0 19920 2314 0 2 0 19920 148 1466 153;9 0 0 0 19 20 20 20 20 0 9 0 0 0 0 9 0 0 0 0 0 0 0 0;13 33 33 34 0 53 54 54 0 33 13 33 0 0 0 13 33 0 0 0 13 14 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4218 760 1450 1 2691 1262 62 424 1172 66 1 5 1305 33245 7 65 3774;0 0 0 0 21 22 0 0 0 0 0 0 0 0 0 0 0;0 0 33 0 0 0 0 33 33 0 0 33 34 0 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 3161 1122 85 2889 2201 300 55 74 1 4 9;0 45 46 46 0 0 0 0 0 0 1 2;0 53 54 54 33 33 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;608 155 5499 44;0 0 29 0;33 0 33 33;1 1 1 1; -;31639 2343 4300 108 263 5742 6 852 450 4334 6 150 32 544 179 586 3265 1159 6888 3 423 59 0 2 0 8777 3201 255 5;17 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;13 33 33 34 33 33 0 33 34 0 0 33 0 33 0 33 34 0 33 0 33 0 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1734 42 3757 2242 1914 45 3 4555 1914 8 11 815 3 541 583 59 57 74 487 18 11 494 338 0 2 0 612 295 42;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 1 2 2;33 34 21 22 22 33 34 0 33 0 0 33 0 25 26 0 0 0 0 33 33 34 13 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;79 20890 11678 11395 1784 1736 6 10 1788 274 798 60 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 33 33 33 33 33 0 0 21 22 22 22 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;24272 1219 13977 179 0;11 0 0 0 0;33 33 33 0 0;1 1 1 1 1; -;14 157 81338 1253 225 33 16 157 81338 1253 225 572 4636 0 2 0 33 4944 5;0 9 10 0 0 0 0 9 10 0 0 0 0 0 0 0 1 2 2;0 13 14 0 0 0 0 13 14 0 0 33 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14419 50271 70 12 70 103 326 1 4 9;23 24 0 0 0 0 0 0 1 2;29 30 33 34 34 0 9 0 1 2;1 1 1 1 1 1 1 1 1 1; -;102 8501 8018 3033 2024 101 6513 886 3512 24 1 8501 8018 100 1 961 110 100 0;0 7 8 7 8 8 0 0 0 0 0 7 8 0 0 17 18 0 0;33 11 12 11 12 12 0 0 33 0 0 11 12 33 0 31 32 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9913 240 5479 3 613 0 14 114 1954 16 1 271 6231;0 0 0 0 0 0 0 17 0 0 0 0 0;33 34 34 0 33 0 0 31 33 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;466 83407 414 28 18;17 18 18 18 18;31 32 32 32 32;1 1 1 1 1; -;41 7233 3 9239 8099 905 0 9239 32 1982 768 41227 1449 13 2 78 2 35 19 2 41 72;19 20 20 20 0 0 0 43 0 0 0 0 0 0 0 0 0 0 0 0 1 2;25 26 26 26 33 34 0 51 0 33 34 33 33 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;17890 2069 423 813 585 240 1256 116 0 2 0 17890 2069 116 326 1359 1491;17 18 0 0 0 0 0 0 0 0 0 17 18 0 0 0 0;11 12 33 33 34 34 33 33 0 0 0 11 12 33 34 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2760 8 22 236 402 90 2172 4434 936 1283 621 722 21 24 17 781 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 34 33 0 33 33 0 33 33 33 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;236 32 986 1402 11077 89717 3 668 0 15 10565 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 0 0 0 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2413 146562 39882 10 20 156 1 4 79 9;0 0 0 0 0 0 0 0 0 0;0 0 0 33 34 33 0 21 22 22;1 1 1 1 1 1 1 1 1 1; -;8827 3259 0 1806 793 118 373 305 2505 442 60 0 793 118 1499 5883 363 78 442 60 0 710 12 3727 4301 21 0 96 15 143 60 1 4 9;23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;29 30 0 33 33 0 0 33 33 33 34 0 33 0 33 0 21 22 33 34 0 0 33 34 33 0 0 21 22 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7071 23 2386 25 11 755 6659 0 1 1 4 2 2 2 9 2 2 2 108 60 60 60;9 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0;13 0 33 0 0 0 33 0 0 0 31 0 0 0 33 0 0 0 33 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;571 483 808 39 127 807 1 4 61;0 0 0 0 0 0 0 1 2;33 34 33 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1; -;11 9943 109 1761 96 91 180 1 1048 325 382 1 1048 325 5;0 0 0 37 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 0 0 0 0 33 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;23832 681 2564 80 828 1 4 130 112;23 24 24 0 0 0 1 2 2;29 30 30 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1; -;49 1064 1003 48 713 145 1217 12 2046 0 1421 13 2387 15715 145 1217 632 1 49 1064 1003 48 713 261 2 1513 134 363 0 12 363 762 2985;45 46 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 46 46 46 0 0 0 0 0 0 0 0 0 0 0;33 34 34 34 33 33 34 33 34 0 33 0 0 33 33 34 33 0 33 34 34 34 33 33 0 33 0 0 0 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;105 7 52 7 129 0 5222 21 391 125 3 18716 0;0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 0 0 0 0 0 33 34 0 11 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 179 675 4081 5577 80999 14638 6714 68223 14927 6714 0 17576 49041 16 2095 1 31546 1 6431 1 44 1 1531 1 23087;0 0 0 9 0 29 30 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 13 33 37 38 33 0 29 30 0 33 33 0 33 0 33 0 33 0 33 0 33 0 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3498 2 111;0 0 17;33 0 31;1 1 1; -;406 12045 7 955 572 887 3 1042 2 2423 271 255 5;0 0 0 0 0 0 0 0 0 17 18 0 0;33 33 34 34 33 33 0 33 0 31 32 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 10503 1521 365 866 1575 866 16 1 102 137 10503 1521 365 866 1575 866 1 1585 10503 1521 365 866 1575 866 44 26 37 1 111;0 0 0 23 24 0 0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 23 24 0 0 0 0 0 0 1;0 21 33 29 30 33 34 0 0 33 21 21 33 29 30 33 34 0 33 21 33 29 30 33 34 33 0 33 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1539 4351 0 2 0 237 8168 7072 17 5917 449 3770 333 792;0 0 0 0 0 0 0 0 0 17 0 0 0 0;33 34 0 0 0 33 33 33 0 31 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;457 1409 2796 3 2427 8 331 10 27348 8 154 11 89 11 70 902 33 3 266 1 4 209 132;0 0 37 0 0 0 0 0 37 0 0 0 0 0 0 0 0 0 0 0 17 23 24;33 33 33 0 33 0 33 34 45 0 0 33 34 34 0 0 33 34 33 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;139 284358 2 125924 144 1 96009 31 1748 6 284358 2 125924 1711 6 284358 2 125924 1711 202 6 5164 6 4528 6 6114 3 1711 40 135 30 2803 202;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6;0 33 0 33 0 0 9 33 0 0 33 0 33 33 0 33 0 33 33 34 0 21 0 21 0 33 0 33 33 33 34 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;294604 0 27385 0 368 0 13191 0 276695 0 158756 0 473 0 85871 0 87742 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 0 33 0 21 0 0 0 33 0 21 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;160 2507 280 570 1490 282 1 4 61;0 0 0 0 0 0 0 1 2;0 33 34 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1; -;9796 1936 3745 250 767 1133 1 4 9;17 18 0 0 0 0 0 1 2;31 32 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;122 3020 3 53121 2082 8 64280 2082 8 74115 18343 2082 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 21 33 0 33 34 0 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14862 988 4344 640 2945 13514 8171 3571 275 2899 1566 13460 1 841 69 1 1368 5;0 0 9 0 0 0 0 0 0 0 17 18 0 0 0 0 1 2;33 33 13 33 0 33 33 33 34 33 31 32 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;35211 1 4 61;0 0 1 2;31 0 1 2;1 1 1 1; -;1294245 2 2794 670 2 3025 3902 2 92 567 98 1177 379 3174 342 107 2 2781 1432 449 2 9871 75 2 92 2 649 1331 62 3987 5 66;21 0 0 0 0 0 0 0 17 18 18 18 18 18 18 18 0 0 0 0 0 9 10 0 9 0 17 18 0 17 18 0;0 0 33 34 0 33 34 0 31 32 32 32 32 32 32 32 0 21 33 34 0 13 14 0 13 0 9 10 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2034 1466 237 6 2034 1466 237 147 1 123 1 469 712 974 0 2 0 460 43 6390 469;17 18 18 0 17 18 18 0 0 0 0 23 24 0 0 0 0 1 2 0 0;13 31 32 0 13 31 32 33 0 33 0 29 30 0 0 0 0 1 2 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 17467 54 22 30 4724 48 1 15022 14356 4165 1 6052 1 4111 300 5;27 28 28 28 28 28 28 0 0 0 0 0 0 0 1 2 2;0 35 36 36 36 36 0 0 33 33 0 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 122 705 26870 0 6721 1355 496 1355 0 4278 36273 1355 16 37 6 87 6 44 6 24177 1355 6 157 27 519612 7886 119 28 18 2 395 5;0 23 24 24 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 18 0 1 2;0 29 30 30 0 33 33 29 30 0 33 33 33 0 33 0 31 0 33 0 33 33 0 13 14 0 31 32 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2475 3 3753 165 82 162 2819 83 1092 20 156 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 0 0 0 0 0 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9847 214 10 0 49 14588 48 0 3 2672 55 579 0 566 0 14 1513 655 2 1513 5 16;0 0 0 0 33 34 34 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;33 0 0 0 33 34 34 0 0 33 0 33 0 0 0 0 33 33 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11 13 422 9024 1315 591 162 1543 136 5662 236 13 124 746 6 555 6003 162 1543 135 2027 7292 13 124 746 8 13 422 638 59 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;55 56 56 33 0 0 33 34 0 33 0 0 0 33 0 0 0 33 34 0 33 0 0 0 33 0 33 34 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 187 525 949 3 2608 2243 692 2918 10 992 3 6 11418 64 1738 2040 2875 13 29 2608 355 6 765 7836 11 1687 20536 62 7647 530 66 3 2608 62 2608 5074 12 4435 6 348 135 10 13 29 426;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 34 0 33 33 33 33 33 34 34 0 0 0 33 34 33 33 34 33 0 0 33 33 0 33 33 0 33 33 0 0 33 0 33 0 0 33 0 0 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 9252 2116 6183 1900 3836 446 16 1 9252 4295 2116 446 10252 327 1 4 493 1575 167 446;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;0 33 33 33 33 34 33 0 0 33 21 22 33 0 0 0 1 2 21 22 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;261169 261 2 3611 1980 1208 260 6 5464 1142 260 8 1980 1142 8 5464 1142 44 6 5464 3601 37 6 14699 1142 37 6 22995 1142;43 0 0 0 9 0 0 0 23 24 0 0 9 0 0 23 24 0 0 23 24 0 0 0 0 0 0 23 24;51 33 0 33 33 33 33 0 29 30 33 0 33 33 0 29 30 33 0 29 30 33 0 33 33 33 0 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14459 75 792 1192 456 4027 223 0 2 0 918 0 474 0 759;9 10 0 0 0 0 0 0 0 0 0 0 0 0 0;13 14 33 0 33 34 0 0 0 0 33 0 33 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5687 50 50 17142 2299 1909 6183 286 0 63402 6183 3193 6269 4595 369 1 573 664 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;27 28 28 31 33 33 33 33 0 0 33 33 34 34 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;184689 2 111;0 0 17;0 0 31;1 1 1; -;1942561 75940 0 1942561 842 0 28896 0 2 0 3107;0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 0 0 21 0 0 0 1;1 1 1 1 1 1 1 1 1 1 1; -;14592 4945 343 1 102 2 7827 38 34 14592 4945 269 294 2085 176 383 2706 2528 343 1 34 819 71 5;0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;39 0 33 0 33 0 33 34 13 39 0 33 34 33 0 33 33 33 34 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 10 401 174 8 298 173 30 530 590 46856 221 8 1558 5420 3 5603 32 18730 685 3 2978 526 1117 8 10 4365 27241 24 17 341 162 173 1082 33 203 17 7 7 7 1 4 9;0 0 0 0 0 0 0 0 39 40 40 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 37 38 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 34 0 33 34 0 47 48 48 0 0 33 0 0 33 0 33 33 0 33 0 0 0 0 45 46 0 0 0 0 0 33 21 22 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;439 13 6493 122 407 4658 4105 4787 8295 6080 103 2121 129 799 52 2469 2 255 2 35 19 2 41 72;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 0 33 0 0 33 0 0 0 0 0 0 0 0 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;466 2775 0 5448 0 97731 46 22 96 5411 12350 3664 0 31 95 0 2 0 942 154 19;17 18 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;31 32 0 33 0 27 0 33 34 0 33 33 0 33 34 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4831 4775 1202 1216 0 975 14777 20 0;0 0 0 0 0 0 0 0 0;33 0 33 34 0 33 33 0 0;1 1 1 1 1 1 1 1 1; -;31176 950 7106 4640 905 3317 199 2 277;9 9 10 10 10 10 10 0 0;13 21 22 33 34 33 34 0 0;1 1 1 1 1 1 1 1 1; -;1636 105 881 50 512 1068 166 101 828 59 1 1408 101 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 7 8 0 0 1 2;33 33 0 0 33 0 0 0 33 0 0 11 12 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4922 1373 8 1783 273 1 4 9 0;0 0 0 0 0 0 1 2 0;33 33 0 33 33 0 1 2 0;1 1 1 1 1 1 1 1 1; -;1942561 2906 3 47;0 0 0 0;0 33 0 1;1 1 1 1; -;52564 1386 4926 40 1 52564 1386 4926 920 40 622 2368 7 85 1 22681 3481 94 5;0 9 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0;33 13 33 33 0 33 13 33 33 33 0 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 206 518 7270 6917 10339 1502 266 883 2 110 2 35 19 2 41 72;0 0 0 7 8 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 11 12 33 33 33 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7058 118 3 782 39703 16588 2960 76 1 405 1 35 19 31 95;0 0 0 0 13 0 0 0 0 29 0 0 0 0 0;33 0 0 0 17 33 34 0 0 1 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3865 11338 278 675 542 15 143 17 64 418 3689 371 442 17 1 1422 217 415 5 1 1572 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2 0 0 0 0;33 33 0 0 33 34 34 0 0 0 0 0 33 34 0 1 2 2 2 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 31359 16 348 4870 8742 9865 8 73 135 12 70 5165 90 21 24 17 1 110 54 115;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 0 0 0 21 22 0 0 33 34 0 33 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2272 1131 16786 115 46 336 1358 3674 336 50 4174 0 140 6052 25353 1 5586 6052 25353 1 6052 8398 1 6052 140 2 73 925 140 406 5;0 19 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 33 34 34 0 55 56 33 0 0 33 0 33 33 33 0 33 33 33 0 33 34 0 33 34 0 27 28 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2884 152 14247 23375 10 20 1 4 9;0 0 0 0 0 0 0 1 2;33 0 33 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1; -;12471 207 78 1 12471 207 78 1229 2 988 7574 48 48 48;9 10 0 0 9 10 0 0 0 0 45 0 0 0;13 14 33 0 13 14 33 33 0 33 33 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 980 48 2820 5386 4130 0 94350 152 24444 82 32728 83 1 188 167;33 34 34 0 0 0 0 21 0 21 0 0 0 0 17 0;33 34 34 0 33 33 0 27 0 27 0 33 0 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;471 7 471 13 848 1316 0 2 0 553 231 29806 1311 349 2 553 0;0 0 0 0 0 0 0 0 0 17 0 0 39 40 0 17 0;21 22 22 33 34 34 0 0 0 31 0 33 47 48 0 31 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;572 5211 368700 129 462354 26 6022 33 12 33 1 4 9;0 23 24 24 24 24 24 0 0 0 0 1 2;33 29 30 30 30 30 30 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;3400 74149 32 29987 3 3033 753 45 8 2899 1687 890 3 7243 13793 19511 129 1297 82653 8 658 1176 3717 118 8 6698 278 12 9732 753 8 1297 937 3579 45 2556 12920 42750 8 57882 118 8 10260 1 361 1 4 554;0 0 0 0 0 0 0 0 0 0 0 0 0 0 31 32 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 0 33 0 33 34 0 0 33 33 0 0 0 39 40 40 0 33 0 0 33 33 0 0 0 0 0 0 0 0 0 0 33 0 33 0 33 0 33 0 0 0 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;683 63 17721 5511 280 1 683 63 17721 5511 280 2131 0 2 0 16099 280 5;9 0 9 10 0 0 9 0 9 10 0 0 0 0 0 17 18 18;13 0 13 14 33 0 13 0 13 14 33 34 0 0 0 31 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;796 7973 15908 14524 796 3694 3 94 4076 32 243 2385 238 29 705 10 12 10 11 145 4089 203 1 3694 43 1 4 47;0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 33 33 0 31 0 21 22 0 21 22 33 34 33 33 34 34 33 34 33 0 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3446 13690 13408 166 42 3 1908 3 511 422 3458 639 8 3584 12 5128 13 145 59 0;0 29 30 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0;33 37 38 0 33 34 34 0 0 0 33 33 0 33 33 34 33 34 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5379 32 1173 1903 30 2593 11338 10 20 2859 57 7 135 10 97 308 619 289 13 2931 7 10 20 2859 60 52 7 62 57 66 2665 2859 32 51628 62 97 74 308 74 619 289 1494 3361 66 514 11 20 2020 60 11 1451 12 453 60 454;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 33 0 33 33 33 34 0 0 0 0 33 34 0 0 33 34 33 0 33 34 0 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 33 0 0 0 33 34 33 0 33 34 33 34 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11 234 9 30 19406 76 56 9102 1478 4999 72706 8 8196 1822 2888 442 17 6 0 11 234 9 30 19406 76 56 9102 1478 4999;0 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 0 0 0 0;33 34 34 0 33 34 0 33 33 33 33 0 33 0 0 33 34 0 0 33 34 34 0 33 34 0 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;19437 770 0 34 5355 37701 5;0 0 0 9 0 0 0;33 33 0 13 33 34 0;1 1 1 1 1 1 1; -;6149 923 113 1087 861 2117 28277 6888 3118;0 9 10 0 0 0 0 0 0;33 13 14 47 48 33 33 33 33;1 1 1 1 1 1 1 1 1; -;12401 4057 4545 2315 2052 398 107 2050 5725 45 2315 700 3 4835 7 660 869 2 1177 2 31 1196;0 0 0 0 0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 34 33 33 33 31 32 33 0 33 33 0 33 0 33 33 0 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;117 5146 25355 327 32449 4495 94 0 34 467 11734 1606 1396 1 6265 7 3001 7 65;0 0 0 0 0 23 24 0 23 24 24 0 0 0 0 0 0 0 0;0 33 0 0 33 29 30 0 29 30 30 33 0 0 33 0 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11289 2109 2718 3 4628 4249 639 1338 1 5013 396 1 504 396 1 4675 193 396 23 15428 193 199 25;5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6 6 0;33 33 33 0 33 33 33 33 0 35 36 0 0 33 0 0 33 34 0 33 34 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 24087 4929 1607 78754 1028 428 146 3 810 40 147 84 1 4 9;0 17 0 0 0 0 5 6 0 0 0 0 0 0 1 2;0 31 33 33 0 33 33 34 0 33 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 540 2278 1169 5 1491 126 262 8083 805 1440 86 11 1680 1 4 9;0 0 0 0 0 0 0 0 0 0 23 24 0 0 0 1 2;0 0 33 33 34 33 0 33 34 33 29 30 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16290 143 1192 2500 159 230 223 0 2 0 918 0 474 0 759;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 0 0 33 34 34 0 0 0 0 33 0 33 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;27819 254 11196 799 11 1742 1 71570 1 114 204;0 37 38 38 0 0 0 0 0 1 2;0 33 34 34 33 34 0 27 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;139 3398 69 5283 144 297 4599 5736 343 0 9617 14694 10148 0 1191 1055 2849 0 136170 1 69 131 1 1337 5;0 9 19 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 13 33 34 0 33 34 33 33 0 33 21 22 0 33 33 34 0 0 0 31 32 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6485 23561 20959 1 598 191 3494 5;17 0 0 0 9 1 2 2;31 15 33 0 13 1 2 2;1 1 1 1 1 1 1 1; -;3618 30263 3 22 0 5125 118 18379 391 12 527 9536 21 1 188 634;0 37 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 0 0 33 0 33 0 33 34 33 0 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;100007 51 9183 220 97 129 181 658 522 105296 17093 2946 22884 343 36 1608 1239 486 36 100007 62 338 1302 2946 516 1056 58 66 1 919 343 2 11036 2769 268 5 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 0 0 0 0 0 0 0 1 2 0;21 0 33 33 34 34 34 27 28 33 21 33 21 22 0 39 40 40 0 21 0 31 32 32 32 32 32 0 0 33 33 0 31 33 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34163 67 1113 18040 709 2 173 115 778 1193 0 22 135 1193 0 2 167 2 35 19 2 41 72;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;27 0 33 33 33 0 33 34 21 22 0 0 0 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;308 11003 190 12469 55738 10669 46 12 7440 152 14129 249 5057 21014 1 876 131 1 4711 268;0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 17 0;21 22 33 34 33 31 0 33 34 0 33 0 33 21 0 33 33 0 33 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;360 1015 1447 49 394 67 297 74 1447 48 1379 0 81 1614 55 4112 85 0 2 0 364 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 33 34 34 0 33 0 33 0 0 33 34 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1226322 0 1693725 0 33283 0 3378 6475 6975 13748 29160 4104 0 9384 3721 2185 5820 3785 9984 3785 0 1482 584 0 23933 0 14 1226322 0 1693725 16 0 2115 7701;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 33 0 33 33 33 33 33 33 0 33 33 34 33 34 33 34 0 0 0 0 33 0 0 0 0 0 0 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2655 2335 2 725 4057 2 2335 2558 1614 55 274 45 68 55 17 2 142 2 35 19 2 41 72;0 0 0 21 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 27 28 0 33 33 33 34 0 33 34 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;913 230 55 127 304 521 1022 602 468 0 384 521 127 304 1022 0 97 562 1827 0 913 3236 55 0 1029 40;27 28 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 0 0 0 0;35 36 36 33 34 34 0 33 0 0 33 34 34 34 0 0 33 34 33 0 31 32 0 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1411 5000 1912 446 16 1 1411 5000 1912 1991 8447 281 2073 1 4 493;0 0 0 9 10 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 13 14 0 0 33 0 33 33 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1551 16094 125 2094 561 1 4 61;0 0 0 0 0 0 1 2;33 34 33 33 33 0 1 2;1 1 1 1 1 1 1 1; -;26612 1086 199 63 218 11081 19463 1173 258 15 589 17 2 0 1253 215 5;9 0 0 0 9 9 9 10 10 0 0 0 0 0 17 18 18;13 33 34 0 13 13 13 33 34 33 34 0 0 0 31 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15 309 10389 3 178 1344 517 63 11858 3 2788 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 34 33 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1874 3511 1081 63 5577 1905 2751 35 4990 23 68 25 36 1874 36 5577 1905 2751 36 35 1 114 69;9 0 0 0 29 30 30 0 0 0 0 0 0 9 0 29 30 30 0 0 0 0 0;13 33 0 0 37 38 38 33 33 0 0 0 0 13 0 37 38 38 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1521 357 1616 1460 87 2001 1074 1460 9216 49272 65591 0 117774 2482 3861 390 16 37 6 87 6 44 6 1616 1107 2115 6 4606 27 36067 75 71762 1237 400 371 2 395 5;0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 18 18 0 1 2;0 33 13 33 33 31 33 21 22 33 0 33 0 29 0 0 0 0 33 0 31 0 33 0 33 33 33 0 13 14 13 14 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2844 27 822 4602 186791 3751 403 346 2 2 3774 277;0 0 0 0 11 17 18 18 0 0 0 0;13 14 13 14 15 31 32 32 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1; -;21163 4740 67 5012 3485 2413 0 2 0 364 5;0 0 0 0 0 0 0 0 0 1 2;33 33 0 33 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;611 1741 29636 2707 1405 3 112359 0 10297 11092 68 692 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 33 33 0 0 0 29 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1293 307126 5319 991 1360 28 18 0 2 0 6086;17 18 18 18 18 18 18 0 0 0 0;13 0 0 39 40 33 34 0 0 0 21;1 1 1 1 1 1 1 1 1 1 1; -;92 5174 544 3 6405 5830 0 14 92 11347 16 2 14408 197;9 0 0 0 0 0 0 0 17 18 0 0 9 10;13 33 34 0 33 33 0 0 21 22 0 0 13 14;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;45272 1 1768 1 188 5652;21 0 0 0 17 0;21 0 33 0 31 33;1 1 1 1 1 1; -;635 556009 46 33 260 8 762 10 77 33 3 61298 0 2 0 452 69 0 2 0 5421 69 5;9 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;13 0 0 0 33 0 33 34 33 34 34 33 0 0 0 33 34 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4318 3288 3 1616 798 44 1338 0 796 2079 54959 0 90243 2654 48302 0 37612 7 146 23017 0 1255 11178 470 0 2 0 93943 546 44 1338 5;19 20 20 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 34 33 33 0 0 33 0 0 0 0 0 0 0 0 0 27 0 0 33 0 0 0 0 27 33 33 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;75117 7706 530 1 75117 10085 1 75117 192 8025 0 2 0 111;17 0 0 0 0 0 0 17 23 24 0 0 0 17;31 33 33 0 21 22 0 31 29 30 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1552 531 908 310 823 11 60845 649 2304 46 8198 1493 234 17569 2 142 2 35 19 2 41 72;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 33 33 33 0 0 0 33 34 0 33 0 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2300 51211 275 16 22 3 4365 10 992 2221 212 3 1 17331 1032 1 167 1 1481 1 743 734 743 734;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;0 27 28 28 0 33 34 33 0 0 27 28 0 0 25 33 0 33 0 1 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;443 183 640 1458 1 4 61;0 0 0 0 0 1 2;29 30 33 33 0 1 2;1 1 1 1 1 1 1; -;14 1470 675 125 944 0 1470 5098 125 944 0 11265 125 944 0 11762 944 4653 16 37 6 87 6 44 6 7366 74 125 944 6 88 3150 288 9031 1107 28 18 2 395 5;0 0 0 0 0 0 0 0 0 0 0 23 24 24 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 9 17 18 18 18 18 18 0 1 2;0 0 0 33 34 0 21 22 22 22 0 29 30 30 0 29 30 33 0 33 0 31 0 33 0 33 0 33 34 0 13 0 0 33 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15898 46 1687 3570 72889 3048 3876 3 4338 5251 1014 0;21 0 0 0 0 0 0 0 0 0 0 0;27 0 33 33 33 33 34 0 33 34 33 0;1 1 1 1 1 1 1 1 1 1 1 1; -;88 27 6950 75 102 53 105 186 1103 39 13 2265 3230 1827 5168 981 1029 1 4 61;9 10 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 14 13 14 33 0 33 0 33 33 34 34 0 33 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9480 14 71929 16 164 3 11848 160 9141 6427 1840 17 1 3841 43 1 4 47;0 0 0 0 0 0 21 0 9 0 0 0 0 0 0 0 1 2;33 0 25 0 0 0 33 0 13 33 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;57828 67 20 1176 4287 5794 799 1 4 9 0;0 0 0 29 37 38 0 0 1 2 0;39 0 0 33 47 48 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1; -;14 1295496 16 1 1295496 37 1 1295496 104 1 1295496 87 1 395 5 0;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;0 0 0 0 0 33 0 0 33 0 0 31 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1166 1006 547 3320;0 0 0 0;35 36 33 33;1 1 1 1; -;27117 0 35848 0 2681 1133 685 0 16400 0 212 40 202 24 17 0 30621 0 6111 93 4673 103 91 574 492 1 4 9;23 24 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;29 30 30 0 33 0 33 0 33 0 0 33 33 0 0 0 0 0 33 0 0 0 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;163485 8137 263 194 3 208 6683 23 29804 60181 11985 25 1 4 61;0 0 0 19 20 20 0 0 0 0 0 0 0 1 2;0 15 33 34 0 35 36 0 21 22 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11107 2437 3 4425 86 321 1450 1 4 61;0 0 0 0 0 0 0 0 1 2;21 22 0 21 22 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1; -;9770 5 0 2 0 34 251 9770 148 36 635 4935 9770 36 9770 280 36 9770 411 5 36 9770 69 36 9770 1648 4476 0 490 490 0 277;1 2 0 0 0 9 0 9 10 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;1 2 0 0 0 13 0 13 14 0 13 13 13 0 13 33 0 13 33 34 0 13 33 0 13 33 34 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;53394 0 313 0 13954 0 284 350 3 1077 45 8 90 70 4166 63 350 3 5358 2 972 2 10596 666 4151 2 10596 300 5 36 10596 3321;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;21 0 33 0 33 0 0 33 34 33 0 0 0 0 33 0 33 34 33 0 33 0 51 0 33 0 51 1 2 0 51 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1786 16632 3008 14159 1434 528 15303 24 363 78 1 4 9;0 0 0 0 23 24 0 0 0 0 0 1 2;29 0 33 0 29 30 0 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 574 226 492 863 1436 3 2773 10 0 2443 19393 1091 11679 165 3 0 6473 1174 10 17460 17460 17460 17460 17460 17460 17460 17460 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 21 22 33 0 0 33 0 0 33 33 0 27 0 0 0 33 33 0 21 22 22 22 21 22 22 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2711 5447 2868 1712 711 528 1230 0 18352 0 20493 0 95854 0 71878 15078 16 37 1 87 1 44 0 2 427 262 318 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 31 33 0 0 33 34 33 0 0 0 0 0 0 0 37 0 0 33 0 31 0 33 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1636 338 3 876 1727 135 9 542 227 20 876 21 1 34 2010 23 124863 25 1424 1 558 1010 5 1424;0 9 0 0 0 0 0 0 0 0 0 0 0 17 18 0 0 0 1 0 1 2 2 1;33 13 0 33 0 0 33 0 0 0 33 0 0 31 32 0 0 0 1 0 1 2 2 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;412 4024 632 257 30 225 1 4 9;17 18 18 18 0 0 0 1 2;31 32 32 32 33 34 0 1 2;1 1 1 1 1 1 1 1 1; -;14 6848 258 11621 334 128 403 16 6848 258 11621 1971 178 260 560 1 2671 2104 261;0 9 10 0 0 0 0 0 9 10 0 0 0 17 18 0 1 2 0;0 13 14 1 33 34 33 0 13 14 1 33 34 31 32 0 1 2 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1640308 2 296 3 207 996 150 1308 259 0;0 0 0 0 0 0 0 0 0 0;0 0 33 0 33 34 34 33 0 0;1 1 1 1 1 1 1 1 1 1; -;32567 0 525 0 2 0 35550 0 36529 0 29755 0 2 0 5786 0 6102;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 0 0 0 33 0 33 0 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1552 1210 107 3290;17 18 18 37;31 32 32 33;1 1 1 1; -;2760 0 3046 309 0 13161 1757 3 1064 54 2258 0 11897 22 13 3175 1636 1840 8 781 21 203 2059 0 1869 147 46 0 17686 7 66579 932 4991 7 65 8 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 0 0 33 33 0 33 34 34 0 33 0 0 0 21 22 0 33 0 0 0 0 33 33 0 0 33 0 27 0 1 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5068 817 207 32 1120 115 140 919 11 20 12 453 17 1 2993 140 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 34 33 33 33 34 33 34 0 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 18094 50 241 16 710 6525 20048 2806 21 29116 1340 3 603 8 626 58 2027 1351 124 17 0 2 0 4387 873 10170 421 0 2 0 3023 4387 873 197;0 0 0 0 0 0 0 21 0 0 39 40 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 9 10 10;0 33 34 34 0 33 34 27 0 0 47 48 0 0 0 33 0 33 0 0 0 0 0 0 33 34 33 34 0 0 0 31 13 14 14;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 116314 32 45558 3 454 2945 2298 275 212 1235 16 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 0 0 21 0 33 0 33 34 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 122 11811 134 12 58 449 83 0 4775 21 20 0 2 0 1909 0 2 0 3908 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 21 22 0 33 34 0 0 0 0 0 0 0 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;218995 3 1061 1 24103 16582 5012 1177 108 1 10808 150 1 2587 10826 1 344 1023 372;21 0 0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 0 0;27 0 33 0 31 32 33 33 33 0 33 33 0 33 33 0 21 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;106510 0 542909 0 311411 0 2 0 7293 2331 8 7557 0 106510 0 542909 0 2 0 321 1396 0 2 0 138805;43 0 0 0 0 0 0 0 0 0 0 0 0 43 0 0 0 0 0 0 0 0 0 0 0;51 0 33 0 0 0 0 0 33 21 0 33 0 51 0 33 0 0 0 33 0 0 0 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;47443 2383 22 6895 6886 20101 25941 4540 3 185 1 185;0 0 0 0 0 0 0 0 0 0 0 1;33 34 0 33 0 33 33 0 0 1 0 1;1 1 1 1 1 1 1 1 1 1 1 1; -;14 1303 101 64 18293 0 110 4928 0 101 64 3754 43649 0 179 3905 7837 43649 0 1958 27939 18293 16 37 6 87 6 44 6 4928 6 2039 81 2691 4928 28 18 2 395 5;0 0 0 0 0 0 0 0 0 0 0 0 31 0 0 0 0 31 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 0 1 2;0 33 0 0 0 0 33 33 0 0 0 33 33 0 0 33 33 33 0 21 22 0 0 33 0 31 0 33 0 33 0 31 32 32 32 32 32 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 7135 716 879 22605 159 16 1 7135 761 5;0 9 0 0 0 0 0 0 9 1 2;0 13 33 34 34 33 0 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1; -;73 2374 1629 687 2135 0 992 507 7162 20388 77 3056;0 0 0 0 0 0 0 0 0 0 0 0;33 34 21 22 33 0 0 33 33 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1; -;49 198 334 440 48;0 0 0 0 0;0 33 33 34 0;1 1 1 1 1; -;14 85 44 16 2 2633 5 36396 34098 7 65;0 0 0 0 0 1 2 0 0 0 0;0 0 33 0 0 1 2 31 32 33 34;1 1 1 1 1 1 1 1 1 1 1; -;10902 14057 220 8 1001 11 509 957 30 594 8 13 1174 101 348 227 12 21 8 13403 348 227 12 21 8 3641 203 17 1 4 9;45 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;53 54 54 0 33 0 0 0 0 0 0 0 33 0 0 0 33 34 0 0 0 0 33 34 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1616 3445 3445 618 1 1616 3445 3445 618 73 78 1 4471 1616 3445 3445 618 1 4471 73 78;45 46 46 46 0 45 46 46 46 0 0 0 0 45 46 46 46 0 0 0 0;53 54 54 54 0 53 54 54 54 33 34 0 1 53 54 54 54 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2807 4630 13948 241 4630 101 2884 8 350 6892 3396 1674 5309 652 118 8 126 4277 3770 12 70 72363 17 2 1353 99;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 0 0 33 34 33 0 0 0 33 33 33 34 0 0 0 33 0 33 34 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;515 113 2329 663 2057 1530 235 71 149 2 2 6772 9894 0;9 10 0 0 0 0 0 0 0 0 0 0 0 0;13 14 33 33 29 30 33 34 34 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3337 118 391 298 2138 19222 1213 871 9159 10 57 0 2 0 4118 253 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6 0;33 34 0 33 34 33 33 34 33 0 0 0 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1166 24655 63 25938 5963 1518 91 1880 1 4 9;9 9 0 9 10 10 0 0 0 1 2;13 13 0 13 14 14 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;515 113 169 287 2;17 18 18 18 0;31 32 32 32 0;1 1 1 1 1; -;1410 27 20400 75 1291 153 198 38 20400 75 5823 297 32 452 286 1291 8972;17 18 18 18 18 18 0 0 9 10 0 0 0 0 0 0 0;31 32 32 32 32 32 33 34 13 14 33 34 0 33 34 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 127 625 1764 1212 893 39438 16 37 1 2621 1 1466 123 1 423 87 1 558 1664 400 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;0 33 34 0 0 33 0 0 33 0 33 0 33 33 0 33 31 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;25035 45683 9210 839 281 297 1628 11902 30095 2 543 6490 192 55;0 0 0 0 0 0 0 0 0 0 9 0 0 0;33 33 0 33 34 33 33 0 33 0 13 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;680 30585 1688 28 727 18;17 18 18 18 18 18;13 33 33 33 34 34;1 1 1 1 1 1; -;5298 22539 159 19798 15 669 1 4 9;0 9 0 0 0 0 0 1 2;33 33 33 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1; -;821 38216 221 845 15 502 520 21 1 4 9;0 0 0 0 0 0 0 0 0 1 2;33 33 0 33 0 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;6337 44193 159 176 1168 13064 149 159 296 5091 3225 6103 18 59 2 71 150 0;17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;31 32 33 0 33 33 34 34 33 34 34 33 33 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;24334 27108 1 2087 5;0 0 0 1 2;13 33 0 1 2;1 1 1 1 1; -;6416 1381 5193 12449 10122 73 11109 9633 1084 1394 1 69 131 1 1 34 1246 5;21 21 22 22 22 0 0 0 0 0 0 0 0 0 0 1 2 2;27 27 28 28 27 33 34 33 34 33 0 31 32 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;47057 812 53 31 291 53 387 5 8 19 35 31 95;0 0 0 0 0 0 1 2 0 0 0 0 0;21 33 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 109173 495 161 436 28 18 116 182 928 244 36 263 325 544 244 2036 174 1 92 2036 116;17 18 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0;31 32 32 32 32 32 32 33 33 34 0 0 33 33 31 32 33 34 0 13 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;39 1705 691 4503 985 522 349 4214 0;0 0 0 0 0 0 0 0 0;0 33 0 33 33 0 33 33 0;1 1 1 1 1 1 1 1 1; -;723 2057 321 161 0 2 0 7318 599;0 0 0 0 0 0 0 1 2;33 34 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;247 205 354 2 167 2 35 19 2 41 72;17 0 0 0 0 0 0 0 0 1 2;31 33 34 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;2174 41644 559 28 18;17 18 18 18 18;13 31 33 33 34;1 1 1 1 1; -;13026 71 5;1 2 2;1 2 2;1 1 1; -;14 68 16 10493 40757 3380 6690 2687 0 178 450 1222 101 10829 1 110 54 115;0 0 0 7 8 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 11 12 33 33 33 0 0 0 0 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 2953 34962 31 2116 1 34962 2116 43 1 4 47;0 0 0 0 0 1 2 0 0 0 0 0 1 2;0 0 0 0 33 1 2 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;398509 10 20 156 1 398509 31 273 1 140 1 1809 1 1526 1 2139 1 704 731 1538;0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0 0 5 6 6;0 33 34 33 0 0 29 30 0 33 0 33 0 33 0 33 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;53 31 291 53 387 5 8 19 35 31 95;0 0 0 0 1 2 0 0 0 0 0;0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1; -;453 365 462 11705 13 29 97 118 4376 21 397 2884 21 1657 11149 350 4500 2509 15 10058 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 33 21 22 22 0 33 34 33 34 0 33 21 22 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2802 18100 0 23 1808 25 0;19 20 0 0 1 0 0;25 26 0 0 1 0 0;1 1 1 1 1 1 1; -;200 29 97 50 3 79 126 12 41 155 2090 17 1 1739 382 1 79 736;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 34 0 21 0 33 34 0 33 0 0 33 34 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;336 1121 49 10638 48 340 485 8458 632 830 1 405 1 35 19 31 95;0 0 5 6 6 9 29 30 0 0 0 29 0 0 0 0 0;0 33 33 34 34 13 37 38 33 34 0 1 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6321 263 5938 509 296 15 124 1 4 9;0 0 0 0 0 0 0 0 1 2;33 33 34 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;2424 447 4424 718 8302 2381 2912 2 244113 5894 2 2262 103 23 4130 25 1 405 1 35 19 31 95;0 17 18 18 0 0 0 0 21 22 0 0 0 0 0 0 0 0 0 0 0 0 0;33 13 31 32 0 33 33 0 27 0 0 33 34 0 33 0 0 1 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;37695 3100 1111 19207 29144 69550 6 233700 276 8 947 6544 24 17 0 758 305 1 4 209 132;37 38 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;45 46 0 33 33 33 0 0 0 0 33 33 0 0 0 33 33 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;338 8 3802 30 1194 2478 56 589 3733 17 2 1714 31964 2 2066 5 204;9 0 0 0 0 0 0 0 0 0 0 23 24 0 1 2 0;13 0 33 0 33 33 0 0 33 0 0 29 30 0 1 2 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1672 771 75 7672 274 4285 3 0 2 0 7877 1302 191 100 0 1 0 7877 31;9 10 10 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0;13 14 14 33 0 0 0 0 0 0 13 33 34 33 0 0 0 13 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1942561 3 47;0 0 0;0 0 1;1 1 1; -;1142 359 241 2060 813 1 22282 148 1566 625 1142 28 18 137 116 71 0 2 0 7222 702 1579;0 0 0 0 0 0 0 0 17 18 18 18 18 0 0 0 0 0 0 9 17 18;33 0 33 34 0 0 21 22 22 22 22 22 22 21 33 34 0 0 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;428 146 4660 300 1348 11 256 1 4 9;0 0 0 0 0 0 0 0 1 2;33 34 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;975 15 143 1 418 56 9522 56 4111 10559 4217 1 33 505 31 2 624 463 5 107 344 372;0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0;33 0 0 0 33 34 33 0 1 33 33 0 1 2 2 0 33 34 34 33 21 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1471 550 38 447 113 1410 27 13428 148 169 1559 8127 1547 150 108 9310 129 42 481 2 3341 116 2 2452 562 5;0 0 0 17 18 18 18 18 18 18 18 18 18 18 18 0 0 0 0 0 0 0 0 1 2 2;0 33 34 13 14 13 14 31 32 32 32 33 33 21 22 33 0 0 33 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1107 20699 16 1 1107 20699 87 1 1107 20699 104 269 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;0 33 33 0 0 33 33 31 0 33 33 33 34 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1014 2949 67 2555;0 0 0 0;33 34 0 33;1 1 1 1; -;173562 35648 12201 12201 1 1652 931 53 31 291 53 387 5 8 19 35 31 95;0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;0 33 33 34 0 33 34 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;30 157 391 125 10 109 15490 33 1709 274 109 5504 33 1709 17 1 4 9;0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 13 33 34 0 0 33 0 33 0 0 31 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9399 626 4080 13048 1255 7787 34 783 2 34 783 2 1905 5 1874;0 0 0 0 0 0 0 0 0 0 0 0 1 2 9;33 33 34 33 34 34 34 34 0 33 34 0 1 2 13;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8216 8973 6937 13567 904 3 15 1851 59 1 4 209 132;0 0 0 0 0 0 0 0 0 0 17 23 24;33 0 0 33 0 0 0 33 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1139 5615 115 14762 123 6 147 6 37 6 751 171 23 68 25 2 5621 2 4 493;21 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 1 2;0 33 0 0 33 0 33 0 33 0 21 22 0 0 0 0 13 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;448 3075 813 1092 7441 117 29 42426 0 146063 38928 378 691 2 247 69;0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0;33 33 34 33 33 0 0 33 0 27 33 33 0 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9309 24769 5279 18394 1951 565 62 92 55348 1951 138 66 0 31 95 0 2 0 942 154 19;17 18 18 18 18 18 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0;31 32 13 33 33 0 0 13 33 31 32 0 0 33 34 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 1982 3066 21 48 38727 1665 82 3121 22 1839 3 1417 8398 83 1 18815 1 167 1 34 911 5;33 34 34 34 34 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 25 26 26 0 27 0 0 0 0 33 0 33 33 0 0 33 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10402 233 352 11 3691 7058 0 170 8386 145 10929 347 859 241 18075 1 188 69 108;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18;33 34 33 33 34 33 0 33 33 0 33 33 0 21 22 0 9 10 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2625 190 19045 619 46327 0 3468 30 4162 164 32 2650 24429 1 188 634;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 33 0 33 0 0 0 33 0 0 33 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5811 67 7011 46 56994 530 29 22025 3 56994 3978 8 90 3777 21 162 29 17 1 524 4887 78;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 0 33 33 34 33 0 33 33 0 0 33 0 0 0 0 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4095 106 386 21725 6723 2977 1909 0 4669 13701 479 353 2522 327 1 389 69 1 588 5;9 0 0 37 0 0 0 0 0 0 0 0 0 0 0 17 18 0 1 2;13 33 34 33 33 0 33 0 33 33 0 0 27 28 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34 626 56 3 20474 11 256 1 4 9;0 0 0 0 0 0 0 0 1 2;21 22 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;6558 371 188224 1663 24851 50764 7441 53134 5366 310 349 630 38 471 53 31 291 53 387 5 8 19 35 31 95;17 18 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;31 32 13 33 21 33 33 0 21 22 33 33 34 33 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7563 457284 11553 43413 5311 379 446 1 7877 75 3661 7805 1 4 493 88 199;0 0 0 0 0 0 0 0 0 0 11 11 0 1 2 9 10;0 33 0 0 33 34 33 0 13 14 33 15 0 1 2 13 14;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5347 638 534 0 836 264 5347 638 534 291 0 836 1026;11 0 0 0 0 0 11 0 0 0 0 0 0;33 21 22 0 33 34 33 21 22 33 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;3277 3 22230 11 256 1 4 9;0 0 0 0 0 0 1 2;33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1; -;14 1329 13837 3296 446 16 1 1329 13837 3296 1 4 493;0 9 0 0 0 0 0 9 0 0 0 1 2;0 13 0 33 33 0 0 13 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 15956 305519 2 1399159 26 11736 16 15956 23 60224 25 149 2462 0 18177 85 0 30884 0 11736 0 4209 23 20081 25 872 1612 1434 62 305519 2 1399159 26 11736 66 14 1236 0 307 0 37 0 1396 16 2 553;0 23 24 24 24 24 24 0 17 0 17 0 0 0 0 0 0 0 0 0 0 0 17 0 17 0 23 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;0 29 30 30 30 30 30 0 31 0 31 0 33 33 0 33 34 0 0 0 33 0 31 0 31 0 29 33 33 0 0 0 0 0 33 0 0 33 0 33 0 33 0 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;75359 0 60062 0 164160 0 23890 0 474 0 313 0 6039 0 7993;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 0 21 0 33 0 33 0 33 0 33 0 9 10 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 275 11179 5135 39618 30102 2604 3 1023 5611 5238 2611 2 255 2 35 19 2 41 72;0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 33 13 33 33 34 0 33 33 33 34 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;617 2234 7757 0 3883 51871 0 3295 99487 0 2296 153060 23 57 25 2 167 2 35 19 2 41 72;9 0 29 0 21 22 0 21 22 0 21 22 0 0 0 0 0 0 0 0 0 1 2;13 33 37 0 27 28 0 27 28 0 27 28 0 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 2 0 3844 25944 6212 0 15427 13 864 0 1702 517 1653 0 446 587 45 0 170 0 2 0 789 2284 299 191 1303 0 2 0 789 1412 5;0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 1 2 2;0 0 0 0 0 31 0 33 0 33 0 0 0 33 34 33 0 33 33 0 0 33 0 0 0 13 33 33 34 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;150 16948 46 21180 13187 429 57679 101 3779 23327 7711 1 749 2321 1 21180 4842;0 39 0 17 0 0 0 0 0 0 0 0 0 0 0 17 18;33 33 0 31 0 0 33 0 0 33 33 0 33 33 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;37009 194 10307 9014 2512 2 210 2 35 19 2 41 72;37 38 38 38 0 0 0 0 0 0 0 1 2;45 46 46 46 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;53480 42519 183 72829 327 2 37243 100 10795 5 490 490 0 14 1671 398 170 791 16;0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0;13 33 34 0 0 0 27 33 1 2 0 0 0 0 33 33 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4566 1201 76425 76 10 23 0 25 1106 3796 3 84 0 408 388 2263 33235 642 388 5583 26186 3 692 715 388 2135 2252 255348 1274 1104 388 7082 1 361 1 4 554;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 22 22 0 0 0 0 33 33 0 0 0 0 0 0 33 0 0 0 33 0 33 0 0 33 33 21 33 0 0 0 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;445 437 8 30177 4899 8 288 2444 445 437 2 210 2 35 19 2 41 72;39 40 0 0 0 0 27 28 28 28 0 0 0 0 0 0 1 2;47 48 0 25 26 0 35 36 36 36 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;23 45 25 18714 51 4549 52 7777 844 3361 177 7777 151 3361 23 57 177 12410 151 3361 25 4634 251 7 53 53 2542 3990 5 53 53;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0;0 0 0 33 0 0 0 33 33 0 0 33 33 0 0 0 0 33 33 0 0 0 0 0 33 34 1 2 2 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;210758 0 109578 0 19221 51 0 12728 7 1054 0 36 0 210758;0 0 0 0 0 0 0 0 0 0 0 0 0 0;31 0 0 0 33 0 0 33 33 34 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;96 993 1315 6456 7758 46 7758 196 3 6150 266;0 0 0 0 0 0 0 0 0 0 0;0 33 0 33 33 0 33 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1; -;71 4053 0 2 0 360 38 117 97 129 181 0 2 0 86684 51 13 522 22063 55 0 2 0 73241 1402 49 30471 48 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 19 20 20 0;33 34 0 0 0 33 34 33 34 34 34 0 0 0 0 0 33 34 33 0 0 0 0 27 0 25 26 26 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;77231 8472 2 1533 31 939 100;0 0 0 1 2 0 0;33 37 0 1 2 33 33;1 1 1 1 1 1 1; -;581 292 5693 949 1385 46 3177 8569 1 34 581 610 5;0 0 0 0 0 0 0 0 0 1 2 2 2;21 22 22 33 33 0 33 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1507 17302 850 5789 55 1 1507 17302 850 5789 55 73 78 1 4471 1507 17302 850 5789 55 1 4471 73 78;45 46 0 0 0 0 45 46 0 0 0 0 0 0 0 45 46 0 0 0 0 0 0 0;53 54 0 33 0 0 53 54 0 33 0 33 34 0 1 53 54 0 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6445 13 159 14509 3075 813 5862 15015 327 2 34 712 564;9 0 0 0 0 0 0 0 0 0 1 2 9;13 0 33 33 33 34 33 0 0 0 1 2 13;1 1 1 1 1 1 1 1 1 1 1 1 1; -;8671 33 5614 4825 2477 1444 41916 1 7489 93045 0 8671 0 4268;0 0 0 0 0 0 0 0 21 22 0 0 0 0;33 0 0 0 0 0 33 0 27 28 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;28107 0 173908 0 41396 0 368 0 12076 2 24999 0 31315 0 36 0 61985 14931 0 84485 0 36 0 14103;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 0 21 0 33 0 33 0 0 0 21 0 0 0 33 33 0 33 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;714 20805 27379 30 0 16400 0 56 3 82 714 20805 27379 83;0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 0 0 33 0 0 0 0 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10254 29912 39 129 604 0 2 0 40242 73 78 282;0 0 0 0 0 0 0 0 1 2 2 0;33 33 0 0 0 0 0 0 1 2 2 33;1 1 1 1 1 1 1 1 1 1 1 1; -;1945 81 557 36994 3272 49 2450 54 12577 48 7574 550 7 2744 7 2480 1 677 19 1 1945 78 19;0 19 20 20 20 45 46 46 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0;1 33 34 34 34 33 34 34 34 34 33 33 0 21 0 21 0 0 33 0 1 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2897 791 2 8028 197 2 2 340 27 2897 791 339 150 2752 211;0 0 0 1 2 0 0 9 10 0 0 0 0 0 0;33 33 0 1 2 0 0 13 14 33 33 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 5728 335 48 44997 17272 27805 555 1105 418 2179 264 2 0 19 31 95 0 2 0 1651 590 358 19;19 20 20 20 21 21 0 0 0 0 0 0 0 0 1 2 0 0 0 0 1 2 2 0;0 33 34 0 27 27 27 0 33 34 33 34 0 0 1 2 33 0 0 0 15 16 16 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 3446 20330 48 4529 7794 50427 153 0 6857 2161 118 152 6001 36 3004 2347 36 17279 1 328 268;19 20 20 20 0 17 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 34 33 33 27 0 0 33 0 0 0 33 0 33 34 0 33 0 13 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;35088 148 169 287;17 18 18 18;13 14 33 34;1 1 1 1; -;13611 524 894 9879 12 373 3 603 11 20 6655 17 0 2 0 7627 0 664 0 2 0 605 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0;31 33 34 33 33 34 0 0 33 34 33 0 0 0 0 33 0 33 0 0 0 1 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12816 76 70 2640 0 2420 78 49 2825 7825 48 6083 122 211 323 23 52 25 1 3333 5 0 9781 8059 7 1054;23 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;29 33 34 33 0 35 36 0 0 0 0 33 0 33 0 0 0 0 0 1 2 0 27 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1199 12234 1421 186 1658 10261 0 85 407 338 32 4344 663 115 1819;0 25 26 26 26 0 0 0 0 9 0 9 0 0 0;33 23 24 24 24 33 0 0 0 13 0 13 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8760 63 3497 721 1497 189 2 8760 63 3497 3799 1497 189 0 2 721 1497 189 196;9 0 9 0 0 0 0 9 0 9 0 0 0 0 0 0 0 0 0;13 0 13 33 34 34 0 13 0 13 33 34 34 0 0 9 10 10 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;263 4533 0 324 393 1389 1146 0 4971 7 19311 7 2965 7 22168 0 2 0 92 0 2 0 1157 232 2 303 321 0 2 0 1157 232 0 334 199 0 2 0 956 0 556 0 1157 232;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 33 34 33 33 0 33 0 33 0 33 0 0 0 0 0 13 0 0 0 9 10 0 33 33 0 0 0 9 10 0 33 0 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11436 63 6443 4106 1497 189 36 11436 63 6443 4106 5645 0 2 0 4505 110 3780 199;9 0 9 0 0 0 0 9 0 9 0 0 0 0 0 0 0 0 0;35 0 13 33 33 34 0 35 0 13 33 33 0 0 0 33 34 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1727 373 5064 47813 240 20 124 442 8 293 453 2005 1084 0;0 0 0 21 0 0 0 0 0 0 0 0 0 0;0 33 34 27 0 0 0 0 0 33 34 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2749 6326 0 2 0 34 15718 5;0 29 0 0 0 9 0 0;33 33 0 0 0 33 34 0;1 1 1 1 1 1 1 1; -;1129 2809 2 11742 27 559 2377 1583 120 153;0 0 0 17 18 18 18 18 18 18;33 34 0 13 14 31 32 32 32 32;1 1 1 1 1 1 1 1 1 1; -;370 872 184 551 464 149 33 24 17 0;23 24 0 0 0 0 0 0 0 0;29 30 33 0 33 34 33 34 0 0;1 1 1 1 1 1 1 1 1 1; -;539 773 15 321 94 31 103 422 136 264 1 22092 3798 34319 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 21 22 22 0 0 1 2;0 33 0 33 33 33 33 34 21 22 0 27 28 28 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;316 3005 1034 39 220 224 2 205 354 2 35 979 19 2 41 72;19 20 20 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 0 0 0 0 33 34 0 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;671 50955 193 122 29 15566 23538 3 229 0 2 0 30833 3 2816 0 2 0 188 204;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6;33 33 33 33 34 33 33 0 33 0 0 0 33 0 33 0 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;39 127 691 82 573 5254 59905 4345 83 11 1658 23625 974 10261 1 1925 114435 1 114 204;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 21 0 1 2;33 34 0 0 33 34 0 33 0 0 0 33 0 33 0 13 27 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6469 67 2206 0 23 1808 25;0 0 0 0 0 1 0;21 22 22 0 0 1 0;1 1 1 1 1 1 1; -;14234 2616 240 21 1654 73 9817 10 15 441 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 0 33 33 34 0 0 33 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 1333 7240 6099 3670 6348 15 1335 17 16178 2477 21 84 11828 4763 32 58474 134 11 84 2 2754 36 512 1093 2 1592 258 67 2336 2 2373 78 100 0 2 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 46 46 46 0 1 0 0 0 0 0;0 33 33 33 0 33 0 0 0 33 0 0 0 33 33 0 33 0 0 0 0 53 0 33 33 0 53 54 54 54 0 1 9 10 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 42164 2346 472 2120 4642 16 1 42164 2346 472 2120 4642 44 1 37 0 2 0 322 5;0 17 0 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 1 2;0 31 33 33 0 33 0 0 31 33 33 0 33 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 13 470 1300 12068 3 184 1462 68 1 4 9;0 0 0 0 21 0 0 0 0 0 1 2;21 22 0 33 27 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;412 11 116 2636 56 5504 3 18 24 17 9018 7931 53 53 8847 1200 160 3 356 310 13 29 97 1009 3259 327 59 1 4 9;9 0 0 0 0 0 0 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 0 33 33 34 31 0 33 0 0 33 21 33 34 21 0 0 0 0 33 21 22 22 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16876 3608 1777 1777 3 424 687 0 2 0 4118 253 0;0 0 0 0 0 0 0 0 0 0 5 6 0;35 36 0 0 0 33 34 0 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;324 76 3652 705 15 89 21 1 4 9;0 0 0 0 0 0 0 0 1 2;0 0 21 22 0 35 36 0 1 2;1 1 1 1 1 1 1 1 1 1; -;4927 25223 18900 238 31720 2 9097 80 13596 60 2 315 2 35 19 2 41 72;0 39 40 0 0 0 9 0 0 0 0 0 0 0 0 0 1 2;33 47 48 0 33 0 33 0 33 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1474333 0 30003 757 71 1 1419 1474333 0 30003 1384 1 2177 766 2563 757 5;0 0 0 0 0 0 9 0 0 0 0 0 0 0 1 2 2;0 0 33 33 33 0 13 0 0 33 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 30 2537 3 18 6179 21 6 999 1561 89 1987 6 1490 3 1561 424 93 160 1561 153 1987 24 17 22 5480 1534 122 21 59 96 12 96 937 2537 18 373 0 2 2 0 391 3375 170 738 372;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;33 34 33 0 33 33 0 0 0 33 0 0 0 33 0 33 34 0 0 33 0 0 0 0 0 33 33 21 22 0 33 34 34 0 33 33 0 0 0 0 0 1 2 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2275 1006 458 244 843 123 16 458 741 8 37 8 87 8 44 8 352 259 8 662 150 8 635 45528 1896 28 18 0 2 0 352 71 0 2 0 2251 1931 516 5;0 27 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 0 0 0 0 0 0 0 0 1 2 2 2;0 35 36 33 34 21 22 0 21 22 0 33 0 31 0 33 0 33 34 0 33 34 0 13 27 33 33 34 0 0 0 33 33 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;277 2 877089 2614 26870 2 322 5;0 0 0 23 24 0 1 2;0 0 0 29 30 0 1 2;1 1 1 1 1 1 1 1; -;14 68 16 2 0 2098 2649 31366 0 2 0 857 2649 10884 0 2 0 857 1412 5;0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 1 2 2;0 0 0 0 0 33 35 36 0 0 0 13 33 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1043928 0 36 0 21230 0 368 0 1043928 0 556 0 26225 2 25548;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 21 22 22 0 0 0 33 0 0 0 25;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;125702 3 1526 1 10 20 156 1 15 817 1 2139 0 1 1094 1 3937 31;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 0 33 34 33 0 0 0 0 33 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;126 11 1588 37 12 6064 3 1460 4607 0 164 2060 32 5635 753 231 10 13322 8 1 540 671 0 2 0 111 1722 922 0;0 0 0 0 0 0 0 0 0 0 31 32 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0;0 0 0 33 33 34 0 33 33 0 33 34 0 33 34 0 0 33 0 0 0 33 0 0 0 31 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1773 1784 53056 0 2 0 1053 202 0 2 0 1443 22 236 1443 3 492 232;21 22 22 0 0 0 5 6 0 0 0 0 0 0 0 0 0 0;27 28 28 0 0 0 9 10 0 0 0 21 22 22 22 22 22 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;29396 46 80 31177 1423 758 17879 17 0;0 0 0 0 0 0 0 0 0;35 0 0 33 33 33 33 0 0;1 1 1 1 1 1 1 1 1; -;2521 623 894 1204 1543 359 15022 2017 1543 852 1 770 6 123 6 147 6 128;17 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0;13 21 22 0 0 0 33 0 0 0 0 33 0 33 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8210 0 133988;0 0 0;33 0 21;1 1 1; -;43460 2645 32 21570 6354 77 50 8 1160 43460 166 21570 77 17802 8 77 6318 1 22485 2645 43 1 4 47;17 18 0 17 0 0 0 0 0 9 0 17 0 0 0 0 0 0 0 0 0 0 1 2;31 32 0 33 33 33 34 0 0 13 0 33 0 33 0 0 33 0 13 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1072 829 665 3958 2886 15 143 0 665 3958 2886 671 982 1 4 9;45 46 0 0 0 0 0 0 0 0 0 0 0 0 1 2;53 54 33 33 33 0 0 0 33 33 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1905 6742 81 590 151 3210 4396 16 1 11577 590 772 1 1905 6742 81 590 151 3210 4396 44 1 37 0 2 0 322 5;0 11 12 0 0 0 0 0 0 0 0 11 12 0 11 12 0 0 0 0 0 0 0 0 0 0 0 1 2;0 15 16 33 34 33 0 33 0 0 33 15 16 0 15 16 33 34 33 0 33 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 3951 1342 16 520 4882 667 12689 20302 9845 2 78 2 35 19 2 41 72;0 45 46 0 19 20 45 46 0 0 0 0 0 0 0 0 1 2;0 53 54 0 27 28 25 26 33 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2384 44116 8 850 682 11 22 53 53 483 329 257 3747 1956 39 200 691 29842 7156 1508;7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 40 0;11 33 0 33 34 33 34 33 34 31 32 32 33 33 33 34 34 47 48 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7407 0 519 0 90278 6 0 258372 0 519 0 83823 0 2 0 87452 0 513 0 46932 0 6230 0 23780 0 513 0 69341 0 33851 6 0 313 0 44167 0 27662 6 0 161878 6 0;0 0 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 10 10 0 0 0 0 0 0 0 0 0 0 0 0;53 54 54 54 54 0 0 0 0 33 0 11 0 0 0 0 0 33 0 0 0 33 0 33 0 33 0 13 14 14 0 0 25 26 26 0 33 0 0 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; diff --git a/PaddleNLP/unarchived/chinese_ner/data/test_files/test_part_2 b/PaddleNLP/unarchived/chinese_ner/data/test_files/test_part_2 deleted file mode 100644 index 80519d1e4eb9c88ee4984817db96c89946945669..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/data/test_files/test_part_2 +++ /dev/null @@ -1,500 +0,0 @@ -;76181 115497 3 424 1755 0 36 0 930 4185 2 39 13 300 14188 4929 301 197;0 29 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0;33 37 0 33 34 0 0 0 1 2 0 33 34 33 55 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;68 165 34 42 3 210 1 44 131 1 1337 5 23 2867 7 65 25 0;0 0 9 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0;33 34 34 0 0 33 0 33 33 0 1 2 0 31 33 34 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3256 160 227 17632 4763 22 190 462 385 135 20535 21 13802 614 298 7307 21 13 29 8851 13 916 122 3 487 10887 9828 10 227 1317 1933 135 236 391 659 259 1 4 9;0 0 0 23 24 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 0 29 30 55 56 56 0 0 0 0 29 33 34 0 33 34 0 0 33 34 0 0 0 0 33 0 0 33 0 0 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;431 386 365 8 1556 2532 2370 862 24 8 886 240 24 17 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;21 22 0 0 0 33 33 33 0 0 0 0 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3032 200 26917 844 1005 23 2853 177 2573 25 1 3647 140 406 5 0;0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0;33 34 33 33 0 0 33 0 33 0 0 1 2 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3273 452 7466 229 2486 4998 1 4 61;0 0 0 0 0 0 0 1 2;33 33 33 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1; -;2413 3904 1 709 253 0;0 0 0 0 0 0;21 22 0 1 2 0;1 1 1 1 1 1; -;90219 2085 13 103 1938 0 12641 45 3 82 1776 83 6 1938 0 146 26 6020 0 5433 2 41321 7 645 0 2 0;0 0 0 0 23 24 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 33 34 29 30 30 33 34 0 0 0 0 29 30 30 0 29 30 30 0 0 0 33 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;198 1684 113 1011 1148 1023 1779 416 0 2 2 0 391 3375 170 738 372;0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;33 13 14 33 34 34 33 33 0 0 0 0 1 2 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1532 86386 0 95565 0 1328 2 1752 0 5234 0 22736 4136 26 21256 26 31574 26 11736 3635 3508 927 16 13621 0 86386 0 95565 0 1328 2 1752 0 5234 0 22736 4136 26 21256 26 31574 26 11736 3635 3508 2189 184 927 1 2483 1;0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 0 0 17 0 23 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 23 24 0 0 0 0;0 29 30 0 21 0 33 0 33 0 33 0 0 0 0 0 0 33 0 33 33 34 33 0 31 0 29 0 21 0 33 0 33 0 33 0 0 0 0 0 0 33 0 33 33 34 29 30 33 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2830 241 3 4505 778 579 8 11268 2830 241 3 4505 778 3624 1537 1724;33 34 34 34 34 0 0 21 33 34 34 34 34 0 0 0;33 34 34 34 34 33 0 27 33 34 34 34 34 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15 1082 317 2138 10037 1 4 9;0 0 0 0 0 0 1 2;0 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1; -;85045 20007 3861 278 830 1039 17 1 4 9;0 0 0 0 0 0 0 0 1 2;33 33 0 0 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;1908 1020 1673 784 21296 54 181 332 1534 450 1850 84 0 2 0 168 1093 0 2 0 2517 3241 7593 5 100 0 1 841 77 50 3 7593 100 0 2 0 956 0 556 0 1157 232 21358;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6 6;33 33 33 0 33 0 0 0 33 0 33 0 0 0 0 33 33 0 0 0 1 2 2 2 33 0 0 33 33 34 0 33 33 0 0 0 0 0 33 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;102 79504 1773 12662 4444 29072 340 199 4056 21535 2 247;0 0 0 0 0 0 0 0 0 0 0 17;33 51 33 33 21 22 13 14 33 33 0 31;1 1 1 1 1 1 1 1 1 1 1 1; -;14 38022 16 829 154 0 467 4495 0 100414 2 17919 26 16038 2 66556 26 26081 62 30282 2800 66 0 94 0;0 0 0 23 24 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 0 29 30 0 29 30 0 33 0 33 0 33 34 34 0 33 0 0 33 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;85 7 0 284 82 27672 716 83 63 82 5384 2824 83 8 14567 212 8 7922 3 456 12 2046 30 5325 47589 84 166 638 49 7922 456 8195 905 48 3 1502 2694 10 46 5698 287 32 138 230 1255 1 361 1 4;0 0 0 0 0 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;0 0 0 0 0 13 14 0 0 0 33 0 0 0 33 0 0 33 0 33 33 34 0 21 33 0 0 33 0 33 33 35 36 0 0 33 33 0 0 0 33 0 33 34 0 0 33 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;105 38 330 567 98 1780 67 2366 159 257 34948 420 1274 1094 3393 878 468 1 504 1 4 61;0 0 17 18 18 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 1 2;33 34 31 32 32 32 32 32 32 32 21 33 34 33 33 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1684 82965 1637 4570 159 138 372 28 18 8 14601 1684 82965 1637 4570 159 138 372 28 18 3 123 1 147 1 14601 469 0 2 0 460 43 6390 469;17 18 18 18 18 18 18 18 18 0 9 17 18 18 18 18 18 18 18 18 0 0 0 0 0 9 0 0 0 0 1 2 0 0;21 22 22 22 22 22 22 22 22 0 13 21 22 22 22 22 22 22 22 22 0 33 0 33 0 13 33 0 0 0 1 2 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3254 2206 3266 3 7087 59 39697 74 6586 1262 61619 11830 17523 0 2 0 36915 0 20793 5 0 23 972 25 0;0 0 0 0 0 0 21 0 21 22 22 0 0 0 0 0 17 0 0 0 0 0 0 0 0;0 33 0 0 33 0 27 0 0 0 0 33 33 0 0 0 31 0 21 22 0 0 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4 9 132 8 22 284 57 2311 1692 888 8 4256 13 12625 8 1810 135 1217 3 1117 1 4 209 132;1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;1 2 33 0 0 0 0 33 33 0 0 0 0 33 0 33 0 0 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;167 3713 0 8164 67206 4342 12837 4296 179 2 167 2 35 19 2 41 72;0 0 0 21 22 0 0 0 0 0 0 0 0 0 0 1 2;33 33 0 27 28 33 33 33 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 282139 458 28 18 2 702 172 128;17 18 18 18 18 0 0 0 0;13 13 33 33 34 0 1 33 34;1 1 1 1 1 1 1 1 1; -;129 2989 10726 13 2021 37 1 45 1370 5;0 0 0 0 0 0 0 1 2 2;0 33 33 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1; -;4041 151 5388 804 6583 756 0 2 0 41755 3 2816 0 2 0 188 204;5 6 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6;9 10 33 0 33 33 0 0 0 33 0 33 0 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3103 1319 8507 2 2 1954 440 2 34 559 819 5;0 29 30 0 0 0 0 0 1 2 2 2;33 37 38 0 0 33 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1; -;2921 423 1455 9051 6031 3041 6 33020 293 1612 2060 37 176 423 87 139 12559 27 11920 376 96100 987 359 144 0 2 0 814 436 662 5;0 0 0 0 13 14 0 0 0 0 0 0 0 0 0 0 0 0 9 10 0 0 0 0 0 0 0 1 2 2 2;33 33 33 33 17 18 0 33 34 33 0 33 0 33 31 0 13 14 21 22 0 33 0 0 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;136 3154 11 1742 285 0 1265 206 2 2133 206 2 285 5;0 17 0 0 0 0 0 0 0 0 0 0 1 2;0 31 33 34 33 0 33 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;263 1456 779 508 1 4 61;0 0 0 0 0 1 2;33 33 33 34 0 1 2;1 1 1 1 1 1 1; -;67924 675 3041 423 1237 1 4 61;0 0 0 0 0 0 1 2;21 22 33 33 33 0 1 2;1 1 1 1 1 1 1 1; -;7313 8991 552 3 20838 1382 311 11 256 1 4 9;0 0 0 0 17 18 18 0 0 0 1 2;33 34 33 0 31 32 32 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;88 11 75424 444 3 1136 292 8 10 215 13 29 17 1128 1086 15 589 17 1 4 9;9 0 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 0 33 0 0 33 34 0 0 33 34 34 0 55 56 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 330 299 22477 3697 307 32 44 1 137 22477 3697 2003 71 16 0 2 0 330 1412 5;0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 13 33 35 33 33 0 33 0 21 35 33 33 33 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1548 31119 299 191 5 1 31119 299 191 3083 1 1548 31119 299 191 1303 16 0 2 0 1548 1412 5;0 9 9 1 2 2 0 9 0 0 0 0 9 9 0 0 0 0 0 0 0 1 2 2;0 13 13 1 2 2 0 13 33 34 34 0 13 13 33 34 33 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;234 93 539 539 22 2853 3332 15 136 8 3804 1 6114 1829 6197 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 1 2;21 22 0 0 0 33 33 33 34 0 0 0 29 30 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;31 2268 1 1109 2769;0 0 0 0 0;33 34 0 31 32;1 1 1 1 1; -;13226 32 33258 11 20 454 1 4 61;0 0 0 0 0 0 0 1 2;33 0 33 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1; -;632 115 2639 64 3716 799 1259 311 19541 6818 2089 54 358 1 69 108 1 114 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 34 33 33 34 0 33 33 21 22 22 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;20378 2 876 131 2 1899 5;0 0 0 0 0 1 2;33 0 33 33 0 1 2;1 1 1 1 1 1 1; -;17607 0 20642 20 156 1 4 9;0 0 0 0 0 0 1 2;33 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1; -;102830 11069 3835 1372 19133 1698 0 88 3302 6332 1698 973 2378 22467 1 88 6045 3835 1698 43 1 4 47;21 0 0 0 0 0 0 9 0 0 0 21 22 22 0 9 0 0 0 0 0 1 2;27 33 33 0 21 22 0 13 33 33 33 27 28 0 0 13 21 22 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1594 2504 1204 4038 4703 5756 9253 0 293 527 4659 25133 12511 2 14 1030 5 16 2 952 252 31 438 696 211 2 163 7 1058 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 34 33 0 0 0 0 0 33 33 33 0 0 1 2 0 0 33 0 33 33 33 33 0 33 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7700 249 172 5513 3 908 9285 4557 11 256 1 3160 3141;0 0 0 0 0 0 0 0 0 0 0 1 2;47 0 33 0 0 33 34 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1718 38 7164 532 168 39 312 596 1302 2546 1807 2 7164 532 2 168;0 0 39 40 40 0 0 0 0 0 0 0 0 0 0 0;33 34 47 48 48 33 34 0 33 34 33 0 33 34 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;503 3736 1596 125 954;0 0 0 0 0;33 34 34 33 34;1 1 1 1 1; -;24651 10 568 4743 18322 41332 2898 527 23 180 667 1352 25 1 13048 1859 7478 1 114 204;0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;55 33 34 33 33 33 34 0 0 33 34 33 0 0 55 56 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15162 15516 25747 45497 6 15162 45497 10 15 441 6 2293 951 10 20 613 1 6940 670 3 204;9 37 38 38 0 9 37 0 0 0 0 0 0 0 0 0 0 21 22 0 0;13 33 34 33 0 13 33 0 0 33 0 0 0 33 34 33 0 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1441 2 472 233 1221 16 90 1500 3 10 509 281 21094 274 509 1866 21094 432 1 1045 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 33 34 0 0 0 0 0 0 0 33 0 0 0 33 0 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3811 34 142 54 1245 1 53682 136 1952 1 114 204;0 0 0 0 0 0 0 0 0 0 1 2;33 21 22 0 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;4412 928 1 4412 928 18 1 4412 176731 4366 928 28 18;17 18 18 18 18 18 18 18 18 18 18 18 18;13 33 0 13 33 34 0 13 0 0 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1; -;179 2483 818 67 325 1668 0 306 121 325 819 5513 20 17 1 841 268 1 69 131 1 3792 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 33 34 34 33 0 0 0 33 34 0 0 0 0 33 21 0 31 32 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4741 286 383 3625 414;17 18 18 18 18;13 33 33 33 33;1 1 1 1 1; -;10414 119 36135 3609 262260 9635 9574 1095 5770 3 3829 23 85 25 1 69 255 1 14584 5 1 680 637 5 1 1293 637 5;17 18 21 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 1 2 2 0 1 2 2;31 32 27 0 21 0 33 33 33 0 33 0 0 0 0 21 22 0 1 2 0 1 2 2 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;56822 8931 175 4191 1778 73 110 1472 2101 171 1 4 9;9 10 10 0 0 0 0 0 0 0 0 1 2;21 0 0 33 33 11 12 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;214 36871 164 73 8358 122 3 16701 2 806 2 35 19 2 41 72;0 0 0 29 30 0 0 0 0 0 0 0 0 0 1 2;0 33 0 37 38 0 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 59 59 59 59 59 70607 9987 3 23732 189 0 2 0 3184 2782;0 0 0 0 0 0 9 17 0 0 0 0 0 0 1 2;0 0 0 0 0 0 13 31 0 21 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;336 422 2530 12 868 5767 0 5767 3941 155 20 2 2 559 131 2 2 169 5 0;0 0 0 0 0 0 0 37 38 0 0 0 0 0 0 0 0 1 2 0;0 0 33 33 34 33 0 45 46 0 0 0 0 33 33 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;30 18 5171 8 963 11897 22 13 29 16902 7769 1658 0;0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 33 0 33 33 0 33 34 33 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;9173 1746 11098 43578 11296 2 186 5 2 1308 325 6 1231 6 1118 6 2351 6 861 6 110 449 71;0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 31 33 33 0 33 34 0 33 33 0 33 0 33 0 33 0 33 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;29432 1427 672 173 10777 760 2888 15 143;0 0 0 0 37 0 0 0 0;33 33 33 34 33 0 0 0 0;1 1 1 1 1 1 1 1 1; -;4064 67551 3 47;0 0 0 0;0 33 0 1;1 1 1 1; -;4443 608 53 197940 2 529 12682 0 4443 202 0 4443 4258 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 33 0 55 0 0 33 0 33 34 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;464 584 681 582048 276 2 210 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 22 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1533 1317 7532 88 816 7300 10420 62632 1 15969 633 1 114 268 1 114 5;9 11 0 9 0 0 0 0 0 0 0 0 1 2 0 1 2;21 33 33 13 33 33 0 33 0 33 33 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2385 547 2431 53886 6788 9013 0 6469 161 853 0 2 0 421 499 480 480;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;33 34 34 33 21 33 0 33 33 34 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;192 4960 396 0 31322 75 192 4960 310 2 142 2 35 19 2 41 72;0 0 0 0 9 10 0 0 0 0 0 0 0 0 0 1 2;29 30 33 0 13 14 29 30 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;548 3031 1610 230 49 581 1226 818 67 3145 48 31 130 127 2 6194 61;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 0 0 33 33 34 0 33 0 21 22 0 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;28319 0 2 0 26667 0 20644 0;0 0 0 0 0 0 0 0;51 0 0 0 55 0 33 0;1 1 1 1 1 1 1 1; -;11744 345 409 2135 2741 684 10 1123 20 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 33 34 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;6317 225 11 12580 541 324 2716 3216 3 1 4 9;9 0 0 0 0 21 22 22 0 0 1 2;13 0 0 33 34 27 28 28 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;1145 81 2338 0 36 0 1518 251 62542 4127 1062 5488 39 200 691 8569 3749 2295 42 9539 2117;0 0 0 0 0 0 0 0 9 10 9 17 0 0 0 0 0 0 0 0 0;33 0 33 0 0 0 33 0 13 14 13 0 33 34 34 33 34 33 34 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1626 229718 3 188 204 0 2 0 13 29 42 7015 0 2 0 188 204;21 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 33 34 0 0 0 33 34 34 34 0 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;322 3823 3871 532 116 1 86476 2637 288 192 662 28 18 116 322 3823 3871 532 1256 1 3080 14584 5;1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 9 1 2;1 33 33 0 33 0 0 33 34 33 34 33 34 33 1 33 33 0 33 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4796 3253 2842 11736 12101 623 282 0 12101 623 15 254 2 3253 2842 11736 1350 2 3112 3065 5;0 45 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;9 33 34 34 21 22 33 0 21 22 33 34 0 33 34 34 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;797077 46 120 311 3 6966 67 1966 120 2 489 11223 5 1 504 1 4 61;21 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 1 2;0 0 33 34 0 21 22 22 22 0 1 2 2 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;95877 1 1 14879 3 47;0 0 0 0 0 0;33 0 0 33 0 1;1 1 1 1 1 1; -;49 3476 32438 55 1187 201 48 1 49 3476 32438 55 1187 201 48 37 1 49 3476 32438 55 1187 201 48 44 1 1685 2762 884 1418 154856 57 1 1237 400 1 8102 516 1 2932 1242 5 0 13887 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 22 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0;0 47 33 33 34 33 0 0 0 47 33 33 34 33 0 33 0 0 47 33 33 34 33 0 33 0 33 0 0 0 0 0 0 33 34 0 33 33 0 1 2 2 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;963 1254 39 1339 173 21 8 8385 762 228 7 154 8 13037 228 1 11 217 846 1021 1 595 217 132;0 37 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 33 34 0 0 33 0 0 0 0 0 33 0 0 1 2 2 2 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4285 2893 164962 8 29584 12635 5272 198 38 471 97 4499 438 1 2940 10 155352 2 28473;0 0 0 0 21 22 22 0 0 0 0 0 0 0 0 0 0 0 0;21 22 22 0 27 28 28 33 34 33 34 33 33 0 33 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6010 4596 62 34 1917 4740 3 1035 66 1 4 253;0 0 0 9 0 0 0 0 0 0 1 2;33 34 0 13 33 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;284 11639 75 29872 103562 175 63 3174 17644 15 109 707 1 4 9;0 9 10 9 10 10 0 0 0 0 0 0 0 1 2;0 13 14 13 33 0 0 33 0 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;83047 21084 14093 18825 46 11198 22 1839 363 1672 0 90 4775 2995 17 1 69 131 1 34 1246 5;21 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;27 0 27 33 0 0 0 33 33 34 0 0 0 0 0 0 31 32 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;123543 22578 7 65 3 339 196 1 14 136 141 212 32632 16 912 1342 7496 345 1 6072 912 7496 345 1 912 1342 136 264 128;0 0 0 0 0 0 0 0 0 0 0 0 21 0 5 6 0 0 0 5 6 0 0 0 5 6 0 0 0;39 33 33 34 0 33 33 0 0 33 34 0 27 0 9 10 25 33 0 9 10 25 33 0 9 10 21 22 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 3671 27 16 14 116 3671 1865 3796 76 897 813 52 42 6 1110 27 4035 75 2206 1865 119 28 18 116 16 1 34 1865 637 5;0 9 10 0 0 0 9 23 24 24 0 0 0 0 0 17 18 18 18 18 18 18 18 18 0 0 0 9 1 2 2;0 13 14 0 0 33 13 29 30 30 33 0 33 34 0 13 14 13 14 33 31 32 33 34 33 0 0 33 34 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1963 1026 54 14621 560 12225 6 1963 1026 54 14621 560 12225 73 78 6 2178 73 78 0 163 7 2178 7 65;45 46 46 46 46 46 0 45 46 46 46 46 46 0 0 0 0 0 0 0 0 0 0 0 0;53 54 54 54 54 54 0 53 54 54 54 54 54 33 34 0 1 2 2 0 33 0 1 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 121 1490 20147 191 5519 31873 83 10 82 35523 3 6636 83 1 485 5924 131 1 6192;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1;0 0 33 21 22 33 0 0 0 0 33 34 34 0 0 25 26 33 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5938 3632 85 2586 2645 1951 0 6150 12 453 5646 2948 1 725 3065 5;45 46 46 0 0 0 0 0 0 0 0 0 0 1 2 2;53 54 54 33 34 33 0 33 33 34 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1007 2722 13 13863 818 865 11808 865 517 639 68 2 191 573 517 639 68 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 0 33 33 34 33 0 33 34 34 0 9 10 10 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;18944 175 384 913 3521 906 1884 8 2312 10034 5398 191 1884 2 92 3443 499;9 10 10 10 10 0 0 0 0 0 0 0 0 0 9 1 2;31 32 32 0 33 33 33 0 33 33 33 34 33 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 26127 0 56163 5520 16 137 307 1 927 1 44 1 100 1 26127 0 56163 5520 538 872 184 201 2 2566 1533 31;0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 0 0 0 1 2;0 31 0 0 0 0 21 33 0 33 0 33 0 33 0 31 0 0 0 33 29 30 33 0 1 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;683 27 39 13 169 107 14128 10069 225 33 6 683 27 39 13 169 107 10006 10069 91 180 1 12005 1231;17 18 18 18 18 18 0 0 0 0 0 17 18 18 18 18 18 0 0 0 0 0 0 0;31 32 32 32 32 32 0 33 0 0 0 31 32 32 32 32 32 33 33 0 0 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;933 13 384 122 407 484 998 15319 46 81 13551 152 5521 11 2757 8 334 1279 805 2723 1 102571 1 12406 69 2 1066 0 13450;9 0 0 0 0 0 37 38 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0;13 0 33 0 0 33 33 34 0 0 33 0 0 21 22 0 33 0 33 33 0 0 0 1 2 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2568 13800 1531 46 338 2639 937 10580 1303 68606 8368 0 2 0 3384 2709 19 5 0 2 0 2561 90 135 1872 1047 0 23 3794 3373 3794 25 3262 2 0 23 0 1624 2 0 1624 25 2661 2539;9 0 0 0 9 0 0 9 0 23 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 33 33 0 13 0 0 13 33 29 33 0 0 0 1 33 34 0 0 0 0 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1985 614 385 497 41 3 42 160 3921 75404 191 10 20 156 135 10 56 3921 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 0 33 34 34 34 0 33 33 0 33 34 33 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 426 6309 9367 9851 10786 3405 0 9367 25320 1608 13384 0 94 9367 4575 12 782 1608 4653 87 2001 16 37 6 87 6 44 6 63951 26 10688 436 6 157 27 2477 620 1505 1000 436 28 18 2 13592 1997;0 0 0 0 0 31 32 0 0 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 18 18 18 0 21 22;0 0 33 31 0 39 40 0 31 39 0 33 0 33 31 39 40 40 40 33 31 33 0 33 0 31 0 33 0 21 0 33 33 0 13 14 27 28 0 33 34 33 34 0 27 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;28632 11 89 11 110 160 8142 160 5302 3 1 4 9;9 0 0 0 0 0 0 0 9 0 0 1 2;13 33 34 34 33 0 33 0 13 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1101 13985 158 366 2912 1101 11106 39 13 5203 137 806 1759 144 139 1486 2 78 2 35 19 2 41 72;0 0 0 0 0 45 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;53 21 33 33 34 53 27 33 34 34 21 33 33 0 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;285088 3190 108819 0 36 0 23781 2991 4430 59 16919 2991 4430 59;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 33 0 0 0 0 0 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;780 3724 2 7504 75 9621 271 175 1265 99 23 141391 414 25 1 92 5540 26 3823 26 1551 1 92 761 5;0 0 0 9 10 9 10 10 0 0 0 17 18 0 0 9 0 0 0 0 0 0 1 2 2;0 33 0 13 14 33 34 0 33 0 0 21 33 0 0 13 33 0 33 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13287 19803 136 3 33 678 30 225 17 1 4 9;9 37 0 0 0 0 0 0 0 0 1 2;13 45 0 0 33 34 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;13064 159 230 168 1029 57 1 4 61;0 0 0 0 0 0 0 1 2;33 34 34 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1; -;14 655 16 39 13 244 7022 11623 49 8729 251 8946 67 152 11080 3 457 48 2501 5417 59 1 8729 8946 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 1 2;0 33 0 33 34 34 37 33 0 25 26 26 0 25 26 26 26 0 33 33 0 0 1 2 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 49847 251 26787 16 2818 548 2 2835 2 2079 0 2828 222 440 49847 20836 11241 301 1 49847 238809 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 0 0 33 0 33 0 33 0 0 0 33 33 33 33 34 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2493 101 151 353 1692 3 101 1649 101 13 29 97 305 91 180 17 1 4 9;0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 1 2;33 34 33 0 33 0 0 33 34 21 22 22 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5567 1317 522 8816 0 410 150 10633 515 3245 557 42 80769;0 0 0 0 0 0 0 0 9 0 0 0 0;33 33 34 33 0 33 34 33 13 0 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;550 38 635 7384 521 554 468 176 361 1 504 1 4 61;0 0 9 0 0 0 0 0 0 0 0 0 1 2;33 34 13 33 33 33 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1808 2388 51 0 2176 12 527 4349 2420 264 164 3 1495 126 1584 1742 1361 904 17 2 247;1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;1 33 0 0 33 34 34 0 33 34 0 0 33 35 36 0 0 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5025 43 0 54766 0 2 0 4514 34 197;0 0 0 0 0 0 0 1 2 2;33 0 0 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1; -;11081 27 1041 2858 3159 400 46190 28 18 2 277;17 18 18 18 18 18 18 18 18 0 0;21 22 22 22 22 22 22 22 22 0 0;1 1 1 1 1 1 1 1 1 1 1; -;10991 3856 1121 19 0 1121 201 2 325 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;35 36 21 22 0 33 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 3702 1013 12719 1464 16 1 3702 1013 12719 1464 37 1 3702 1013 12719 1464 104 1 3702 1013 12719 1464 87 1 395 5 0;0 29 30 30 29 0 0 29 30 30 29 0 0 29 30 30 29 0 0 29 30 30 29 0 0 1 2 0;0 37 38 38 33 0 0 37 38 38 33 33 0 37 38 38 33 33 0 37 38 38 33 31 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 3 16704 3377 19191 21 8 11 106 29 122 97 21 8 22 236 217 10 12 10 5885 3 619 2444 21 84 373 2277 15848 24 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 34 33 0 0 0 21 22 0 0 0 0 33 34 0 33 34 34 0 0 21 22 0 0 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4770 167 258 334 128 3097 62910 14705 3263 2975 261 0 2 0 12 1626 2 34 495 71 5;0 0 0 0 0 0 0 0 19 0 0 0 0 0 0 0 0 1 2 2 2;33 13 14 33 34 0 21 33 33 33 33 0 0 0 33 34 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;564 628 7044 235 119 28 18 0 2 0 1755;17 18 18 18 18 18 18 0 0 0 0;13 0 0 33 34 33 34 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1; -;283 29 97 21 8 11 356 1369 366 10 6937 6937 3 15 441 203 1 1739 382 1 79 736;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 0 33 34 33 33 34 0 0 0 0 33 0 0 33 34 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 27288 9875 6 12 3727 309 22784 824 21 13 103 6 2469 101 24 60 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 34 0 33 0 33 34 0 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7061 3086 23 6550 530 29 731 25 0 1 4 130 112;0 0 0 0 0 0 0 0 0 0 1 2 2;33 34 0 33 33 34 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;5148 2586 1312 13 29 24924 217 22 160 21 659 1378 109 5466 475 442 8 22 455 89 11 109 254 8 10027 350 1378 96 5480 96 478 424 160 59 22 16066 160 1636 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 33 34 0 33 33 34 35 36 0 33 0 0 0 0 0 0 0 33 34 0 0 0 33 0 33 0 33 0 0 33 0 0 21 22 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;79325 95042 3 47;0 0 0 0;0 0 0 1;1 1 1 1; -;6216 50 36623 499 62 21961 54 2258 66 0 4784 1 4 9;19 20 20 20 0 0 0 0 0 0 0 0 1 2;25 26 26 26 0 33 33 34 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1290 184 331 10 385 2046 5 8 8 362 89 11 1850 2128 867 8 8 999 70 2453 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 33 34 0 21 22 0 0 33 33 34 33 33 33 0 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1261 14923 980 57 0 923 14923 0 1261 14923 0 29655 53 31 291 53 387 5 8 19 35 31 95;0 0 0 0 0 33 34 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;55 56 33 0 0 33 34 0 55 56 0 13 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11277 450 1135 740 375 27587 3 904 701 830 7 5096 1167 1740 7 305 24248 319 24 60 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;33 0 0 33 0 0 0 33 34 0 0 0 33 34 0 33 33 33 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 29 32 511 29 205 354 534 23 57 2 25 0 2 0 35 31 95 0 2 0 618 136 5;19 20 20 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;47 48 48 48 48 33 34 33 0 0 0 0 0 0 0 33 33 34 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2333 27 394 5488 69 696 153;17 18 18 18 18 18 18;31 32 32 32 32 32 32;1 1 1 1 1 1 1; -;4373 45215 2 14 1030 5 16 2 952 252 31 438 696 211 2 163 7 1058 7 65;0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 0 1 2 0 0 33 0 33 33 33 33 0 33 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1179 2102 394 3785 307 1 1179 2102 394 3785 104 1 1179 2102 394 3785 87 16 1 788 600 105 2 1473 2 151;0 9 0 0 0 0 0 9 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0;0 13 21 22 0 33 0 13 21 22 0 33 0 13 21 22 0 31 0 0 33 0 33 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 3691 206 74181 48 2038 46 23000 3 623 23 936 177 5344 66 0 2 0 3001 3788 2222 171 0 2 0 1113 817 14216 0 2 0 2975 261 100 0;27 28 28 28 28 0 0 21 22 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 29 0 0 0 0 0 29 0 0 0;0 33 34 34 0 33 0 27 28 28 0 0 0 33 0 0 0 0 33 55 33 34 0 0 0 33 33 34 0 0 0 33 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;275512 70652 3 185 1 185;0 0 0 1 0 1;33 27 0 1 0 1;1 1 1 1 1 1; -;1858 8643 1583 10 81 3456 899 3 314 3 13 29 3101 5507 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 33 34 33 34 34 25 26 26 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 34 754 101 414 16;0 17 18 18 18 0;0 31 32 32 32 0;1 1 1 1 1 1; -;19206 6486 572 16805 6 126 373 93 2007 3 1361 2343 1 4 9;21 22 0 0 0 0 0 0 0 0 0 0 0 1 2;27 28 33 33 0 0 21 22 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5245 1026 10048 4268 1 5245 1026 10048 1108 1 5245 1026 10048 47;0 0 0 0 0 21 22 22 22 0 9 10 0 0;0 0 29 33 0 27 28 28 28 0 0 0 29 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;85 97 431 181 656 113 6811 18 10807 2170 1632 18 928 108 1592 2446 682 270 138 833 481 0 2 0 248 1301 833 5;0 0 0 0 17 18 18 18 18 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 13 14 33 33 13 14 14 33 33 34 33 33 34 33 33 33 34 0 0 0 33 34 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;360 38 34 166 2568 15548 1676 4302 6472 353 2819 1 11283 339 1 34 11283 5;0 0 9 0 9 31 0 0 0 0 0 0 0 0 0 1 2 2;33 34 13 0 13 39 33 25 33 0 0 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;94 2122 242 2976 571 23 22 539 22 1839 3 406 46567 25 23 588 10927 55 25 0 2 0 14390 142 3 2816 0 2 0 142 31 197 0 2 0 956 0 556 0 1157 232;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0;21 22 0 33 33 0 0 0 0 33 0 33 33 0 0 31 21 22 0 0 0 0 33 33 0 33 0 0 0 1 2 33 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;21664 45 523 50 241 960 2234 0 82 167052 39373 83 2379 4493 354 0 141943 1 33293 39373 0 2 0 1249 7 65;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;27 33 34 33 34 0 33 0 0 0 33 0 33 33 34 0 0 0 35 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;88 60906 4966 13902 1807 847 0 4966 1143 2224 20 1121 0;0 9 0 0 0 0 0 0 0 0 0 0 0;13 13 33 0 33 33 0 33 33 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;159 270 383 611 2418 1 2087 5;0 0 0 0 0 0 1 2;33 34 34 33 34 0 1 2;1 1 1 1 1 1 1 1; -;2248 2666 4083 1313 1 4 253;11 12 12 12 0 1 2;15 16 16 16 0 1 2;1 1 1 1 1 1 1; -;1756 1907 230 1580 62 7665 66 1 894871 10013 1 1655 3115 3168 5;0 0 0 0 0 0 0 0 21 0 0 1 2 2 2;35 36 36 36 0 33 0 0 27 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4207 5244 69425 3 47;0 0 21 0 0;33 33 27 0 1;1 1 1 1 1; -;5489 4808 8648 18959 24;37 0 0 0 0;45 0 33 0 0;1 1 1 1 1; -;7021 281 99 46 2479 26856 3568 0 1317 12069 88314 2 3782 131 2 1899 5;17 18 18 0 23 0 0 0 0 0 0 0 0 0 0 1 2;31 32 32 0 29 33 33 0 33 33 0 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;521 2388 844 206 96 1067 1277 3 0 33 22 58 3121 246 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 33 33 0 0 0 21 22 0 0 55 56 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2988 5292 6517 927 1656 2486 8186 698 1 4 61;0 0 0 0 0 0 0 0 0 1 2;33 34 33 33 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;3091 0 197 0 51 51 0 164691 52 0 51 51 0 78;1 0 0 0 0 0 0 0 0 0 0 0 0 0;1 0 33 0 0 0 0 31 0 0 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;25885 1983 404 3 33 24 274 458 2081 17 1 4 9;17 0 0 0 0 0 0 0 0 0 0 1 2;33 33 21 22 22 22 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;31583 220 1230 1428 1 31583 220 1230 1428 37 1 31583 220 1230 1428 104 26 182 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;33 0 21 22 0 33 0 21 22 33 0 33 0 21 22 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;211921 54 704 3 47;0 0 0 0 0;33 0 0 0 1;1 1 1 1 1; -;139 18 144 18593 207 57 651 2229 890 30831 1041 625 0 7654 2409 3894 15084 1 3713 131 1 3792 5;0 0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 31 32 0 0 21 22 0 33 34 0 33 0 0 0 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;28074 7621 82 3723 83 185 0 1843 19474 1275 82 278 2780 1483 2780 83 23 68 25 2 2 34 1590 5 0 1038 169 1590 3688 5145 0 835 77 50 300 2853 235 411;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 17 18 18 18 0 0 0 0 0 0 0 0 0;27 33 0 33 0 1 0 33 33 0 0 33 34 34 34 0 0 0 0 0 0 1 2 2 0 31 32 32 32 33 0 33 33 34 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2981 115 17045 3 47;0 0 0 0 0;33 34 33 0 1;1 1 1 1 1; -;14 5044 10009 16 773 11 89 11 20 572 33 3 301 74 141 222 403 84 1 128874 1039 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 0 33 33 34 34 0 33 33 34 33 0 33 0 33 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1521 288 5782 623 7319 16 37 6 87 6 44 6 5712 7448 6 218 27 4008 75 29743 145 2510 2115 1172 244 2 395 5;0 0 17 18 18 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 18 18 18 18 0 1 2;0 33 31 32 32 33 0 33 0 31 0 33 0 0 33 0 13 14 13 14 33 0 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1901 27 21030 75 198 116 14464 45 384 610 1 1901 1089 116;9 10 10 10 0 0 0 0 0 0 0 9 0 0;13 14 14 14 33 33 33 33 34 33 0 13 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;160 357 1264 8 256 0 4796 0 6039 0 572 29646 2758 2772 17 0 2 0 605 1071 3 1487 0 2 0 605;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1;0 21 22 0 0 0 9 0 33 0 33 0 0 33 0 0 0 0 1 33 0 33 0 0 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1899 876 2 1615 890 77 50 77 296 3 876 383 1093 211;5 6 0 0 0 0 0 0 0 0 0 0 0 0;33 34 0 33 34 33 34 0 33 0 33 34 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5868 0 93456 0 19908 0 197254 0 98058 0 275364;0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 33 0 33 0 29 0 0;1 1 1 1 1 1 1 1 1 1 1; -;45 1105 143 1909 82 76461 83 1454 46 5970 58 4724 6748 1024 4189 2 2 1024 2876 2 2 169 5 0;17 18 18 0 0 0 0 0 0 17 18 0 0 0 0 0 0 9 10 0 0 1 2 0;31 32 32 33 0 21 0 33 0 31 32 33 33 0 0 0 0 33 34 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 504 16 299 2384 715 0 10252 0 615 518 6690 1876 7 220 1891 1 69 108 1 2633 5;0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 33 11 0 0 0 0 33 0 33 33 0 0 33 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;484 5153 3 81792 16779 1 4 61;0 0 0 0 0 0 1 2;33 33 0 21 33 0 1 2;1 1 1 1 1 1 1 1; -;3503 142 186 1 403768 10013 1 1655 3115 3168 5;0 0 0 0 21 0 0 1 2 2 2;33 34 34 0 27 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1; -;4965 1150 2970 81 2636 0 38502 403 139 8489 144 0 2 0 78 398 0 2 0 6903 713 5;45 46 46 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;53 54 54 0 0 0 27 33 0 0 0 0 0 0 33 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 505606 1750 917 16 505606 1750 917 0 1690 5375 7 228 7 154 334 170 40 1 1916 55 40 2 4675 193 201;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 34 0 0 33 34 0 0 0 0 0 0 0 33 33 33 0 33 34 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5155 9803 4134 4287 5155 541 249 22 319 1972 1 4287 5155 1 33 505 31 2 624 463 5 107 344 372;37 0 0 37 38 0 0 0 0 0 0 37 38 0 1 2 2 0 0 0 0 0 0 0;45 46 0 33 34 33 34 0 33 33 0 33 34 0 1 2 2 0 33 34 34 33 21 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14675 433 569 54 1247 704 1855 13499 2 2835 0 14675 433 569 54 1247 704 1855 13499 1 546 1 405 1 35 19 31 95;19 20 20 20 20 20 20 20 0 0 0 19 20 20 20 20 20 20 20 0 0 0 0 0 0 0 0 0;33 34 34 34 34 34 34 34 0 33 0 33 34 34 34 34 34 34 34 0 33 0 1 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2602 42875 4225 2617 1 4 61;0 0 0 0 0 1 2;33 33 0 0 0 1 2;1 1 1 1 1 1 1; -;15287 11169 2225 315 445 437 0 2009 2009 9676 0 2508 3014 435 32 5806 2190 2 210 2 35 19 2 41 72;0 0 0 0 39 40 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 21 33 47 48 0 33 34 34 0 0 33 33 0 33 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8264 4438 3 855 2705 2 34 4438 287 5;0 9 0 0 0 0 17 18 18 0;33 13 0 33 33 0 13 31 32 0;1 1 1 1 1 1 1 1 1 1; -;1482 3003 8 1699 10815 240 3420 56 644 11 73 753 1298 408 8 32488 47081 3587 145 3 2620 73 1659 12483 13 585 685 408 8 2367 13 1180 11688 73 753 1298 642 8 715 121 3587 145 735 11381 3420 56 13 145 8 12483 63 3420 1913 23467 265;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 33 33 0 0 33 34 0 33 34 0 0 0 0 0 0 33 34 33 34 29 30 33 34 33 0 0 35 36 36 33 33 34 0 0 0 0 0 0 0 0 33 0 0 33 34 0 33 0 0 33 21 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;80 4593 5886 56 3721 2185 266 2103 36 3354 5886 56 3721 2185 438 44 201 20653 1471 2136 575 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 33 0 33 34 33 33 0 33 33 0 33 34 33 33 33 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 22485 335 2040 446 16 1 22485 335 2040 844 327 739 281 2073 1 4 493;0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 1 2;0 13 0 0 33 0 0 13 0 0 33 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;32271 7740 2858 2437 1780 267 997 359;17 18 18 18 18 18 18 18;13 33 34 33 29 30 33 34;1 1 1 1 1 1 1 1; -;64 497 109 6 150 348 58 11 82 10745 83 221 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 0 33 0 0 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;898 68 121 2349 1174 6502 2707 7373 68 8 541 9168 68 1487 388 62 57 66 1442 27299 121 24537 450 27499 3741 8 1438 1060 2891 30 24537 146417 56 3 1 1 1 1 1 1 45 2059 2530 45 7161 1 361 1 4 554;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 33 33 33 34 33 34 0 0 0 0 33 0 0 0 0 0 0 0 33 0 33 34 0 0 33 33 0 33 21 0 0 0 0 0 0 0 0 0 0 33 0 33 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7957 4096 1 10867 7957 31951 4096 27262 620 904 6713 33 0 7288 4096 0 2 0 111;31 32 0 31 32 0 0 0 0 0 0 0 0 0 0 0 0 0 17;39 40 0 39 40 33 33 33 34 0 33 0 0 33 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;160 617 280 8 3470 13 29 35975 8567 9803 15 124 17 0 2 0 4228 382;0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0;0 13 33 0 0 33 34 55 33 34 0 0 0 0 0 0 1 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6217 71 257 647 223 1 4 9;17 18 18 0 0 0 1 2;31 32 32 33 34 0 1 2;1 1 1 1 1 1 1 1; -;575 7755 3 356 845 202 15 347 60 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;33 33 0 0 33 34 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;3475 2215 82 1475 577 2409 83 3779 1273 221 241 278 39 13 470 1612 1856 2 247 69;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 33 0 33 34 34 0 0 33 34 33 34 33 34 34 33 34 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14517;0;21;1; -;2871 314 179 1553 3 547 323 2437 8 402 314 276 243 4901 305 1793 194 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 0 21 22 22 0 33 33 0 33 34 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 20639 33 12637 16 10246 27486 2481 1 10246 27486 2481 1 268228 125552 3 2816 2 1065 573;0 11 0 0 0 11 12 12 0 11 12 12 0 0 0 0 0 0 1 2;0 33 0 33 0 15 16 16 0 15 16 16 0 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12792 737101 49 73 10377 48 1 279 19;21 22 33 34 34 34 0 23 24;0 0 0 27 28 0 0 29 30;1 1 1 1 1 1 1 1 1; -;24180 3655 8632 40 1 809 40 1 35 40 1 809 2919 141 2332;0 0 0 0 0 0 0 0 0 0 0 5 0 1 2;0 33 33 33 0 9 33 0 33 33 0 9 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 522 5300 44 1197 46 46964 49931 898 13607 3039 43336 1 35 4242 1 114 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 21 22 33 0 33 7 0 33 34 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;126 11 42 699 155 21372 432 17 893 634 134 14329 8 50 309 50 309 3 1918 527 155 59 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 33 0 55 0 0 33 34 0 33 0 33 34 33 34 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;31174 0 368 0 901284 0 291053 0 2 0 19265 0 24491 0 2 0 15572 0 23780;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 0 0 0 0 0 0 33 0 33 0 0 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;26961 641 219 1646 33 1 4 9;0 0 0 0 0 0 1 2;33 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1; -;110 7439 11303 14634 221 11 15416 1 4 9;23 24 0 0 0 0 0 0 1 2;29 30 33 33 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1; -;27884 43479 0 2678 0 20182 0 6972;0 0 0 0 0 0 0 0;55 33 0 0 0 27 0 33;1 1 1 1 1 1 1 1; -;113 1356 864 5 725 12523 212 22 864 700 1807 71 149 32 128 263 125;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 34 33 34 33 0 0 0 33 33 33 34 0 1 2 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2280 672 173 1211 2629 6 0 2280 1145 672 173 1211 2629 6 0 2280 1145 106 10497 1211 196 2 707 5 0;9 0 0 0 0 0 0 9 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0;13 33 34 25 26 0 0 13 33 33 34 25 26 0 0 13 33 33 34 0 33 0 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;79 127 248 511 173 21 366 10 3988 3 16309 10 15 441 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 33 34 34 34 0 33 34 33 0 33 0 0 33 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;187 763 24128 1812 5484 1392 3 4122;0 0 0 0 0 0 0 0;0 33 33 33 33 34 0 0;1 1 1 1 1 1 1 1; -;10572 237 3228 312 93 143 183 24 60 1 4 9;17 18 0 0 0 0 0 0 0 0 1 2;33 33 33 34 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;119 845 44 458 845 170 40 1 877 51 5667 1 1240 51 152453 16875 30678 3949 1 23 44 2013 51 1942561 25 2 916 2093 163 7 702 2759 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0;33 33 34 33 33 33 33 0 33 0 33 0 33 0 0 33 0 33 0 0 33 33 0 0 0 0 1 2 33 0 1 31 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 146 68 16 16544 12168 2928 0 358 3715 164 0 15644 1265 6768 50 2902 11 1551 223 0 1222 540 959 813 8 2844 358 3715 164 299 191 1303 0 2 0 2844 631 5;0 0 0 0 9 10 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 1 2 2;0 0 0 0 13 14 14 0 33 34 0 0 33 33 0 33 34 0 35 36 0 0 0 33 34 0 13 33 34 0 33 34 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9368 9996 0 15701 122 3 580 1137 3 58 33697 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;53 54 0 33 0 0 0 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15167 27 185650 9945 1208 944 259 371 15 124 17 23 137 116 74 719 1238 1183 8 1195 648 74 544 578 74 1815 2314 25 0 2 0 1774 884 224 23 13 199 712 729 1256 25;17 18 18 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0;13 14 0 33 33 34 33 34 0 0 0 0 21 33 0 33 33 34 0 33 33 0 33 33 0 33 33 0 0 0 0 1 2 2 0 33 34 33 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;187 17707 3 11732 3623 0 3706 483 134 11 3 17 1 4 9;0 21 0 0 0 0 0 0 0 0 0 0 0 1 2;0 27 0 33 33 0 31 13 14 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 15039 11225 43 16 134096 320 451 0 13489 0 9815 0 74428 0 24744 0 2 0 16653 0 62429 0 2 0 195907 19 4258 1980 75 0 2 0 170 40 0 4350 11225 43 1093 211 0 2 0 956 0 556 0 1157 232;0 1 2 2 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0 0 0 0 0;0 1 2 2 0 27 0 0 0 25 26 26 0 1 0 0 0 0 0 31 0 33 0 0 0 55 33 33 33 0 0 0 0 33 33 0 1 2 2 33 33 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;187 4946 1343 13873 3917 369 483 125 3 621 1 1 895 13873 3917;0 0 0 0 0 0 0 0 0 0 0 0 9 0 0;0 33 33 0 33 33 34 33 0 33 0 0 13 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1050 109 54060 10260 16560 118 1621 12 305 45863 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 33 0 0 21 22 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1112 414 1269 345 2066 1 170 40 1 4 61;0 0 0 0 0 0 0 0 0 1 2;0 33 33 34 34 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;439 13775 3453 18147 4845 42 0 1585 332 170 14388 1161 567 880 1006;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 33 33 33 34 0 0 33 0 33 33 33 34 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 72295 1800 28 18;17 18 18 18 18;31 32 32 32 32;1 1 1 1 1; -;3093 2460 2345 7871 0 10007 53 31 291 53 49 2206 54 1145 1079 1759 419 0 794 48 53 1031 53 387 5 8 19 35 31 95;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;27 25 26 33 0 33 0 21 22 0 0 33 0 33 33 33 0 0 25 0 0 33 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;105 38 2058 1867 4856 2 315 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;22908 75 2619 415 153 63 148 2619 415 153 1093 406;17 18 18 18 18 0 0 0 0 0 0 0;13 14 31 32 32 0 0 31 32 32 33 33;1 1 1 1 1 1 1 1 1 1 1 1; -;2932 570 2134 138 68 1 1 172 8583 2134 1 2134 1732 1 138 1930 1 3079 68 5 3171 7 65;17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0;31 32 33 34 0 0 0 21 22 33 0 33 33 0 33 33 0 1 2 2 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;531 71;0 0;33 34;1 1; -;895 74 650 65748 3246 23 32218 19361 25 9793 123 2 34 3135 733 5 0;9 0 9 17 18 0 11 12 0 0 0 0 1 2 2 2 0;13 0 13 31 32 0 15 16 0 33 33 0 1 2 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;627 0 2 0 1045 15 3284 13 29 11 44 3 5070 0 2 0 11916 11507;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 0 33 0 33 33 34 0 33 0 33 0 0 0 33 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15012 6157 1 2252 1380 4268 809 40 1 1464 1920 2252 712;0 0 0 5 6 0 0 0 0 0 0 0 0;11 21 0 9 10 33 9 33 0 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 301 365 2301 1577 63 6823 11 717 3368 32 32532 1243 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 33 34 35 36 36 33 34 34 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;340 6274 847 116 71 36 340 558 41417 116 71 2 9976 637 5;9 0 0 0 0 0 9 17 18 0 0 0 1 2 2;13 33 33 33 34 0 13 31 32 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2010 237 81 13 1936 151 353 3 4684 10 91 8 27555 80 669 1 53465;17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;31 32 33 34 34 33 0 0 33 0 0 0 25 0 33 0 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;233 103 3142 2063 477 1 233 103 3142 2063 477 37 1 233 103 3142 2063 477 104 26 182 0 2 0 111;0 0 0 23 24 0 0 0 0 23 24 0 0 0 0 0 23 24 0 0 0 0 0 0 17;33 34 33 33 34 0 33 34 33 33 34 33 0 33 34 33 33 34 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;980748 2579 3 492 1 980748 3 492 1 980748 137 636 0 2 0 2326 202;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6;0 21 22 22 0 0 21 22 0 0 21 22 0 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2230 10932 1 360 3881 3517 2230 10932 0 615 654 0 104 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;33 33 0 33 33 34 33 33 0 21 22 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;958 82 194 2388 523 83 190 2301 363523 152 16153 0 1215 6047 2388 1394 1 1 6220 2642 2 635 411 128;9 0 39 40 40 0 0 0 21 0 11 0 0 0 0 0 0 0 0 0 0 9 0 0;13 0 47 48 48 0 33 34 27 0 33 0 33 25 26 33 0 0 21 22 0 13 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3218 2 0 2521 322 3358 0 2521 322 939 0 2521 337 3358 0 2521 337 939 0 295425 337 939 0;0 0 0 9 1 0 0 9 1 0 0 9 0 0 0 0 0 0 0 0 0 0 0;33 0 0 13 1 33 0 13 1 33 0 13 21 22 0 21 22 22 0 21 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5670 2738 2498 711 22902 1 5670 2738 2498 711 22902 37 1 5670 2738 2498 711 22902 104 26 182 0 2 0 111;0 23 24 24 0 0 0 23 24 24 0 0 0 0 23 24 24 0 0 0 0 0 0 0 17;33 29 30 30 33 0 33 29 30 30 33 33 0 33 29 30 30 33 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;36334 3208 324 2716 3216 8877 1 36334 3208 324 2716 3216 8877 44 926;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 27 28 28 33 0 33 33 27 28 28 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6742 260 6354 323 1461 2805 208 2 647 609 7 2210 869 2 647 609 2 31 1196;11 17 18 18 0 0 0 0 0 0 0 0 0 0 17 18 0 0 0;33 31 32 32 33 34 33 0 31 32 21 22 33 0 31 32 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 2074 22878 42771 4258 6 1847 1724 48 22332 2 1486 7 1689 7 62704 36 2765 43 36 1847 1724 36 1537 1724 36 1847 995 36 14246 36 14198 36 35 170 40 36 28633 1847 5;0 0 0 0 0 0 0 0 0 21 22 22 22 22 22 22 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 33 34 34 34 0 33 34 0 27 0 0 0 0 0 27 0 1 2 0 33 34 0 33 34 0 33 34 0 33 0 33 0 33 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15676 30 2349 325 45 332 527 2474 1468 3 657 8 10 2349 450 3 13 29 8932 3 82 1225 359 83 388 22 275 10 5155 3 27841 75 8 9168 1291 22 275 5155 2604 9803 311 2004 121 57 7 52 1041 8 3231;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 0 0 0;33 0 33 33 0 33 34 33 33 0 33 0 0 33 0 25 26 26 33 0 0 33 34 0 0 21 22 0 33 0 33 0 0 0 33 21 22 45 46 46 46 0 0 33 34 34 34 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;220 2311 835 559 37 5922 962 200 38 46501 489 169 2298 275 662 244 128;0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 0;0 33 33 33 33 0 0 33 34 0 31 32 32 32 32 32 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7874 3 1776 34062 6 227 21 135 317 42 7821 0 2 0 1932 0 2 0 790 1889 8567 912;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;27 0 0 0 0 0 0 33 34 0 33 0 0 0 33 0 0 0 0 9 33 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;165022 45670 23427 125982 40 1 165022 45670 23427 125982 334 40 1 165022 45670 23427 125982 52 7 154 2 9746 193 292;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;27 51 33 0 33 0 27 51 33 0 33 33 0 27 51 33 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1200 888 459 1369 11 4850 1040 1742 8423 862 24 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;0 0 33 33 0 21 22 0 0 33 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;187 109 33 105 38 22 113 50821 263 120 714 2741 1910 270 2596 125 3 621 2 656 113 297 32 71 323 1056 58;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 18 18;0 0 0 33 34 0 0 0 21 22 22 33 34 33 34 33 0 33 0 31 32 32 32 32 32 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;503 2794 121 47499 2824 108133 0 39314 1856 154 353 2522 327 4691 2 342 5 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;33 34 0 33 0 33 0 33 34 0 0 27 28 33 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;139 19458 5236 144 1 70991 537 125547 31 1748 6 19458 5236 1711 6 19458 5236 1711 202 6 5164 6 4528 6 6114 3 1711 40 135 30 2803 202;0 9 10 0 0 21 0 0 0 0 0 9 10 0 0 9 10 0 0 0 0 0 5 0 0 0 0 0 0 0 5 6;0 13 14 0 0 21 0 0 33 0 0 13 14 33 0 13 14 33 34 0 21 0 21 0 33 0 33 33 33 34 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1103 125 954 343 25900 1 336 2 2909 0 2 0 421 499 480 480;0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;33 33 34 33 33 0 0 0 33 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2232 292 164 3 38560 23731 938 62 788 962 1876 224 66 2 534 31 95 2 205 354 2 41 72 231 5 1125;19 20 20 20 20 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 1 2 0;33 34 34 34 34 34 34 0 33 0 33 0 0 0 33 33 34 0 33 34 0 1 2 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1062 13 1046 27431 8359 17195 0 5521 121 25144 9978 15685 30396 11102 46611 5770 1 342 10783 5;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;13 0 33 33 33 33 0 0 0 33 0 0 17 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;125 944 4653 3 1709 2585 0;0 0 0 0 0 0 0;33 34 33 0 33 33 0;1 1 1 1 1 1 1; -;1548 122 42 39314 1856 152 9460 0 49191 75 336 42 5766 22531 152 2416 1 114 984 1 114 5;9 0 0 0 0 0 0 0 9 10 0 0 0 0 0 0 0 17 9 0 1 2;13 21 22 33 34 0 33 0 13 14 0 0 33 33 0 0 0 31 13 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1166 6178 358 98731 5038 18384 9350 329 5378 14538 208 2 2 49 45 7948 48 360 38 2047 276 0;9 29 30 30 0 0 0 0 0 0 0 0 0 27 28 28 28 0 0 0 0 0;13 37 38 38 35 36 36 36 33 34 33 0 0 0 17 18 0 33 34 35 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5 483 2130 1123 20 1 4 9;0 0 0 0 0 0 1 2;33 34 33 0 0 0 1 2;1 1 1 1 1 1 1 1; -;3104 27 33 6700 12052 4802 2845 1 2521 207 1998 258 200 75 1 2521 1771;9 10 0 17 18 23 24 0 9 10 10 10 10 10 0 1 2;13 14 0 31 32 29 30 0 13 14 14 14 14 14 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1293 512 161 257 11 256 296 2 1293 512 161 257 296 347 1 98 174 4868 5;17 18 18 18 0 0 0 0 17 18 18 18 0 0 0 1 2 2 2;31 32 32 32 0 0 33 0 31 32 32 32 21 22 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;459 6391 15857 6937 10 12 10 12 33 3 16776 1 1739 382 1 79 736;37 38 38 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 45 0 33 34 34 33 34 0 33 0 33 34 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7049 7400 6667 5026 2040 59961 17310 1252 62 5626 4964 42910 66 0 10570 81854 3559 178 1065 282 1 7049 7400 6667 5026 2040 59961 17310 1252 62 5626 4964 42910 66 0 10570 81854 3559 178 147 1 7049 7400 6667 5026 2040 59961 17310 1252 62 5626 4964 42910 66 0 10570 81854 3559 178 5985 1285;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 34 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 349 9544 16 44 170 40 1 349 9544 926 1 349 9544 838 2 916 2093 163 7 702 2759 7 65;0 0 23 0 0 0 0 0 0 23 0 0 0 23 0 0 1 2 0 0 0 0 0 0;0 33 29 0 33 33 33 0 33 29 33 0 33 29 33 0 1 2 33 0 1 31 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6272 2591 2443 46 82 46470 71857 83 6272 15 165 17 0 1 81 3647 6272 5 1 6272 406 411;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 34 0 0 33 33 0 33 33 34 0 0 0 0 1 33 0 0 21 22 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2425 945 251 3241 7797 445 270 2999 22539 159 779 1 34 833 5 1 1073 113 833;17 18 0 0 0 0 0 0 0 0 0 0 1 2 2 0 9 10 0;31 32 0 0 0 33 33 33 33 33 34 0 1 2 2 0 13 14 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12546 906 2498 10873 8 688 9118 1294 227 13 644 688 13 644 59 8 330 12257 67912 34482 5458 10500 299 191 312 419 0 2 0 191 573 0;0 0 0 0 0 0 9 10 0 0 0 0 0 0 0 0 9 9 0 9 10 10 0 0 0 0 0 0 0 1 2 0;33 33 0 0 0 0 13 14 0 33 34 0 33 34 0 0 13 13 0 13 14 14 33 34 0 0 0 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13694 28098 3 185 1 185;0 0 0 1 0 1;33 33 0 1 0 1;1 1 1 1 1 1; -;18611 17196 14406 3041 155 122 21 11 910 657 24 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;33 0 0 33 0 21 22 0 33 34 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4459 9941 30 1592 70 240 3787 442 1 4 209 132;0 0 0 0 0 0 0 0 0 17 23 24;37 38 33 34 0 0 33 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1; -;28394 3 156 1 28394 10 20 156 1 28394 3 3123 1 4942 1 1809 2 3647 31 1538 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 33 33 34 33 0 33 0 33 0 33 0 33 0 1 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 10254 3705 48 54 5095 11889 49 2233 888 48 1219 0 2204 20013 791 1 452 69 1 588 5;45 46 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 0 0 33 0 33 0 0 33 0 33 33 33 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2823 240 460 1170 286 684 0 2 0 364 5;0 0 0 0 0 0 0 0 0 1 2;21 22 22 22 22 22 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;4191 567 52 99 213 38 450 3877 599 8080 599 2 247;0 0 0 0 0 0 0 0 0 0 0 0 17;33 34 0 0 0 0 0 0 33 34 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1; -;402 51 40317 57941 76396 10 20 1650 3 3376 40317 57941 76396 10 20 1650 3 3376 30 0 2 0 612 295 42;0 0 13 14 14 0 0 0 0 0 13 14 14 0 0 0 0 0 0 0 0 0 1 2 2;33 0 17 18 18 33 34 33 0 33 17 18 18 33 34 33 0 33 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13158 0 26722 0 40452 0 519 0 62748 0 18626;0 0 0 0 0 0 0 0 0 0 0;33 0 31 0 33 0 33 0 21 0 0;1 1 1 1 1 1 1 1 1 1 1; -;22 1839 524 511 29 97 21 8 15 2554 274 12 58 199 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 0 0 0 0 0 0 0 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16959 622 21366 1 16959 622 21366 37 1 16959 622 21366 104 26 182 0 2 0 111;23 0 0 0 23 0 0 0 0 23 0 0 0 0 0 0 0 0 17;29 0 33 0 29 0 33 33 0 29 0 33 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;384 312 304 745 410 917 19 49 11 3 42 48 1614 55 14 2627 847 16 0 31 95 2 384 745 410 917 2 19856 5 62 198037 7 65 66;27 28 28 0 0 0 0 27 28 28 28 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;33 34 34 33 33 34 33 0 33 34 34 0 33 34 0 0 33 0 0 33 34 0 35 36 33 34 0 1 2 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;50051 0 862980 0 2 0 58833 0 1671625;0 0 0 0 0 0 0 0 0;0 0 0 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1; -;530 38 118 8 137 161 3642 7832 278 8331 3 20424 99 37104 71920 2 56 557 69 0 15547 5312 3 3345;0 0 0 0 0 0 0 0 0 0 0 7 8 0 0 0 0 0 0 0 0 0 0 0;33 34 0 0 21 33 33 33 0 33 0 25 26 33 33 0 21 22 22 0 33 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;102 38 536 98 335 2375 329 257 179 7991 637 7548 116 481 1 1853 637 5;0 0 17 18 18 18 18 18 0 0 0 0 0 0 0 1 2 2;33 34 31 32 32 32 32 32 33 34 34 33 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 76296 48 3 1633 1 629 4804 1 3738 1 1334 224 1 1633 2919;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 27 0 0 33 0 33 34 0 33 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 6169 726 825 28 18 6 92 6169 726 825 28 18 128 277 6 403 6 843 123 6 147;17 18 18 18 18 18 0 17 18 18 18 18 18 0 0 0 0 0 0 0 0 0;13 0 0 33 33 34 0 13 0 0 33 33 34 1 2 0 33 0 21 22 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;143 1492 664 3503 2660 381 140 200 312 417 0 2 0 918 0 474 0 759;0 17 18 18 18 18 0 27 28 28 0 0 0 0 0 0 0 0;0 31 32 32 32 32 33 33 34 34 0 0 0 33 0 33 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2565 1311 1 2068 114 1044 980 1 114 5;17 0 0 0 0 0 0 0 1 2;31 33 0 31 31 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1; -;6460 52429 8811 31 19 6 4589 64612;0 0 0 0 0 0 0 0;33 0 53 54 33 0 33 31;1 1 1 1 1 1 1 1; -;1482 62 85 66 3003 8 7538 20833 9797 45 8 4066 7330 27287 8 5977 7330 28243 8 1222 306 1275 7082 14118 33295 8 18914 145 715 67 145 408 20876 8 9892 46 1600 1201 91520 10 13939 1600 1201 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 0 0 0 0 0 33 0 33 0 0 33 0 0 0 33 0 35 0 0 0 0 0 29 33 0 0 0 0 33 34 0 33 0 33 0 33 34 21 0 33 34 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;540 1133 168 8 2105 594 434 1008 168 14962 0 2 0 1008 159 230 0 2 0 79283 7 65 1 8638 5474 311 0 2 0 956 0 556 0 1157 232;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 0 33 0 33 33 33 0 0 0 0 33 34 34 0 0 0 0 33 34 0 33 33 34 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5736 460 289 7063 9032 3991 46 21249 1384 16213 36 191 816 36 7063 36 5736 460 1 114 268 1 114 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 1 2;33 33 0 33 33 33 0 33 33 0 0 33 34 0 33 0 33 33 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22857 824 1033 560 0 39 1184 224 2 60620 7733 42 2 18954 19;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 34 0 0 33 0 0 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14221 6254 339 4290 391 2700 27 786 379 5051 7250 437907 6925 505 15 124 74 33 12 33 2 33 505 31 2 624 463 5 107;37 38 38 38 0 17 18 18 18 37 38 21 22 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0;45 46 46 46 0 31 32 32 32 33 34 21 22 33 0 0 0 33 34 34 0 1 2 2 0 33 34 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 666 6692 1086 199 1065 469 14 666 6692 1086 199 906 2096 16 11055 73 5112 261;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 0 33 33 34 33 34 0 0 33 33 34 33 33 0 33 34 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 70 309 112990 2149 24 8 12 514 214 1893 904 122 21 1 48761 100 1 110 54 115 100;0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 13 0 0 0 0 0 0;0 0 0 0 0 11 33 0 0 0 0 0 0 0 21 22 0 17 33 0 9 10 10 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;373 89 1214 63 365 462 115 19325 33 24 1 4 9;0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 34 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;2517 1357 1010 3 516 11 1547 24 17 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;142 2696 1 6514 27 1642 718;0 0 0 17 18 18 18;33 33 0 31 32 32 32;1 1 1 1 1 1 1; -;243 785 1 709 253 0;0 0 0 1 2 0;33 34 0 1 2 0;1 1 1 1 1 1; -;22511 0 16591 0 24712 0 2 0 12185 0 1251119 0 78903;0 0 0 0 39 0 0 0 0 0 0 0 0;21 0 33 0 47 0 0 0 33 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;4659 11862 213 70 427145 2156 20 156 60 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;14 18981 16 37 6 87 6 44 6 5328 6 1496 58217 9525 1225 28 18 2 395 5;0 31 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 0 1 2;0 39 0 33 0 31 0 33 0 0 0 21 22 22 22 22 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1589372 0 123 4009 196 36 94 730 3630 0 2 0 6623 43;0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 34 33 0 33 34 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 10348 2491 1792 16 1 44 176 37 1 1688 1 516 1 1236 1 100 1 255 1 632 400 1242 201 2 1920 2691 1920;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 33 34 0 0 33 0 33 0 33 0 33 0 33 0 33 0 33 0 33 34 33 33 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 18316 3839 10790 57 14899 102 38 26381 434 531 230 168 2558 30251 1807 176 19443 878 602 19109 14899 3839 10790 13 434 531 230 2558 2385 3145 14899 15867 2620 57 48 0 14 770;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 21 33 34 0 33 33 34 47 33 34 34 34 34 33 33 0 33 21 22 47 33 33 34 0 33 34 34 34 33 34 33 33 0 0 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;24017 4436 4131 227 94 12 5639 58 250 17 0 2 2 0 391 3375 170 738 372;17 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;21 33 34 0 33 0 33 0 0 0 0 0 0 0 1 2 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;241173 2 69875 2964 33496 2553 250 363 38 118 502 13626 2 2586 214 899 668 2 7978 19;0 0 0 0 0 0 0 0 0 0 0 0 0 19 20 20 20 0 0 0;0 0 33 33 33 33 0 0 0 33 34 33 0 33 34 34 34 0 9 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;40810 4036 1 40810 4036 37 1 410 40810 4036 104 26 182 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;33 0 0 33 0 33 0 33 33 0 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1416 38 179 1871 308 8374 69782 72584 0 200 390 179 5330 194 3 8374 440 1 4317 4662 1 34 299 101 258;0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 1 2 2 2;33 34 0 33 34 11 33 34 0 0 0 0 33 0 0 11 33 0 0 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 226 407 3 761 222 13 29 572 231 3 226 407 3 5001 7 16 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 33 34 0 33 21 22 0 33 0 0 33 34 0 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 13 1922 59445 312 6764 83 5755 495 81 8014 0 2 0 1140 623 197 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 33 0 33 0 33 33 0 33 0 0 0 33 34 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;786 902 760 501 985 2590 44 2501 170 40 1 877 51 5834 23 44 2013 51 1942561 25 2 916 2093 163 7 702 2759 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0;33 34 34 34 31 32 33 33 33 33 0 33 0 9 0 33 33 0 0 0 0 1 2 33 0 1 31 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9309 18492 9129 1035 28 18 92 246 18 770 2 456 2182 5;17 18 18 18 18 18 18 18 18 0 0 1 2 2;21 22 22 22 22 22 13 33 34 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4172 237 42890 183 2369 1219 0 12918 82 42890 83 10526 363 1622 0 36 268 36 297 36 18 1 114 268 1 114 5;17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 1 2;31 32 21 22 33 33 0 33 0 33 0 33 0 33 0 0 21 0 33 0 33 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22352 7497 86 1 22352 7497 86 37 1 260 1 87 1 104 2 34 5697 5;0 23 24 0 0 23 24 0 0 0 0 0 0 0 0 1 2 2;0 29 30 0 0 29 30 33 0 33 0 31 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1386 4252 3344 2725 3370 159 28 18 0 2 0 2251 1931 172 1270;17 18 18 18 18 18 18 18 0 0 0 21 22 0 0;13 0 0 33 34 34 33 34 0 0 0 27 28 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 182158 7 0 61242 7 0 133117 1 4 9;0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 33 0 0 21 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;4894 160 14946 0 42 122 12 122 17 0 2 0 3184 2782;0 0 9 0 0 0 0 0 0 0 0 0 1 2;33 0 13 0 0 0 33 34 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;54160 310409 22192 8 7287 54160 310409 22192 3 123 1 147 1 7287 469 0 2 0 460 43 6390 469;17 18 18 0 17 18 18 18 0 0 0 0 0 9 0 0 0 0 1 2 0 0;13 21 0 0 13 13 21 0 0 33 0 33 0 13 33 0 0 0 1 2 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 644 979 3 3146 193 305 91 28808 17 1 5013 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 0 33 34 33 0 33 0 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3240 1038 3959 3827 49 187 6585 275 11 172 1245 3 1807 1696 48;17 18 18 0 27 28 28 28 28 28 28 28 28 28 28;31 32 31 33 0 35 36 36 36 36 36 36 36 36 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3091 0 197 0 51 51 0 643 0 51 51 0 15260 27 109 751 5023 123 2920;1 0 0 0 0 0 0 0 0 0 0 0 9 10 0 0 0 0 0;1 0 33 0 0 0 0 0 0 0 0 0 13 14 0 33 34 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6290 15 902 0 3100 1474 58 152 28723 1474 507 1500 0 6290 2889 12 10612 2 1353 99;37 0 0 0 29 30 0 0 29 30 0 0 0 37 0 0 37 0 1 2;33 0 0 0 37 38 0 0 0 33 33 0 0 33 33 0 45 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;876 2 0 1242 1164 0 2 0 605;0 0 0 0 0 0 0 0 1;33 0 0 33 34 0 0 0 1;1 1 1 1 1 1 1 1 1; -;178198 32018 2528 16958 30 2457 28518 2581 4541 2315 45 3 571 639 2 14 1030 5 16 2 952 252 31 438 696 211 2 163 7 1058 7 65;0 0 0 0 0 37 38 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 33 33 33 0 45 46 33 33 34 33 34 33 33 0 0 1 2 0 0 33 0 33 33 33 33 0 33 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 2089 3 3294 83 317 117 494 7457 42 839 1948 5030 1 342 10783 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 33 34 34 0 0 0 0 33 34 33 34 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2174 2578 458;9 0 0;13 33 33;1 1 1; -;22733 2009 0 31 95 0 2 0 942 154 19;29 30 0 0 0 0 0 0 0 0 0;37 38 0 33 34 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1; -;958 30648 1384 1822 2797 91 1 4 9;0 9 0 0 0 0 0 1 2;33 34 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;14 642 37082 231 538 54820 6444 3412 6 44075 192 13 417 741 16 37 1 87 1 44 0 2 427 262 318 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 0 33 33 21 22 0 33 33 0 0 33 0 33 0 31 0 33 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;389 837 98 1209 663 67 161 296 403;17 18 18 0 0 0 0 0 0;31 32 32 33 34 34 34 34 33;1 1 1 1 1 1 1 1 1; -;45 467 2638 7700 1139919 0 1222 1220 540 6602 908 1324 772 1 188 119;17 18 0 0 0 0 0 0 0 0 0 0 0 0 17 18;33 34 33 47 0 0 0 33 0 33 33 33 34 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;17285 383 270 15097 333 1499 3909 2 342 5 0;0 0 0 0 0 0 0 0 1 2 0;21 22 22 22 0 33 33 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1; -;542 828 20 2273 1 4 9;0 0 0 0 0 1 2;0 33 0 33 0 1 2;1 1 1 1 1 1 1; -;4995 2212 319 107 1 564 2212 107;0 0 0 0 0 0 0 0;13 33 33 33 0 21 22 33;1 1 1 1 1 1 1 1; -;610 4912 6711 15397 8003 30373 343 2 1388 436;0 0 0 0 0 0 0 0 0 0;33 33 33 33 0 0 33 0 35 33;1 1 1 1 1 1 1 1 1 1; -;14 2510 173 10888 5551 446 16 1 2510 173 10888 5551 530 8447 281 2073 1 4 493;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 34 33 33 0 0 33 34 34 33 33 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2658 3013 117 46 7206 1791 1650 60932 611 429 935 0 23 57 25 0 1 2658 3013 117 46 7206 282 4174 1 3333 5;45 46 46 46 46 0 0 29 0 0 0 0 0 0 0 0 0 45 46 46 46 46 0 0 0 1 2;53 54 54 54 54 33 33 33 33 34 33 0 0 0 0 0 0 53 54 54 54 54 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1419 312 1298 13728 608 2846 1084 152 2630 89 634 2056 0 8415 40261 23 1488 25 1 69 131 1 489 5;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 21 22 22 33 0 0 0 0 0 33 34 0 33 33 0 0 0 0 31 32 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9394 10700 52 66116 55 272 1068 3916 294 0 1044 682 18798 4415 5596 1 2122 2641;45 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;53 54 54 33 0 33 34 33 33 0 33 34 33 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;90 3 14713 15497 21 90 3 497 2 7235 2745 4715 2 2745 5;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 33 0 33 34 34 0 33 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1317 11362 516 883 2 2 2 5646 1005 23 2103 25 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 33 0 0 0 33 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 664 491;17 18 18;33 34 0;1 1 1; -;2365 90 3 210 6542 0 611625 4056 2048 551 1860 852 1 2280 65549 92 873 1 114 204;0 0 0 0 0 0 0 0 0 0 0 0 0 9 9 10 10 0 1 2;33 34 34 34 33 0 21 33 33 34 33 34 0 13 13 14 14 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1864 12 2046 1087 3 3310 1 7807 1633;0 0 0 0 0 0 0 0 0;33 33 34 33 0 33 0 33 33;1 1 1 1 1 1 1 1 1; -;16479 1814 58 2272 6312 3289 82 4416 83 2 14909 6312 3289 1 16479 6312 3289 1095 1 14909 1 581 292 6312 3289 1 2488 3837 5;0 0 0 0 39 40 0 0 0 0 0 0 0 0 0 39 40 0 0 0 0 0 0 39 40 0 1 2 2;33 0 0 33 33 34 0 33 0 0 21 22 22 0 33 33 34 33 0 33 0 33 34 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 1131 1078 83 110 1871 487 11 3733 17 0 2 0 5785 666 737 5431 31;0 23 24 0 0 0 0 0 0 0 0 0 0 1 2 2 2 2;0 29 30 0 33 0 0 35 36 0 0 0 0 1 2 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;312 304 521 339 2223 349 508 3311;0 0 0 39 40 40 0 0;33 34 34 47 48 48 33 33;1 1 1 1 1 1 1 1; -;14 352 195674 14427 188682 1947 2056 4159 12457 3775 3617 104 88348 16 37 1 87 1 44 0 2 427 262 318 5;0 0 0 9 21 0 0 11 11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 13 27 35 36 33 15 33 33 33 0 0 33 0 31 0 33 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;157 2141 303 219 33 8 96 15 828 17 1 642 1 31390 1 114 204;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 33 33 0 0 0 21 22 33 0 0 0 0 55 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 21945 3461 3541 3541 865 787 16 1 21945 3461 3541 3541 865 787 44 1 37 0 2 0 322 5;0 0 0 0 37 38 38 0 0 0 0 0 37 38 38 0 0 0 0 0 0 1 2;0 33 29 30 45 46 46 0 0 33 29 30 45 46 46 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1942561 3 633 1 790 47 0;0 0 0 0 0 0 0;0 0 33 0 1 2 0;1 1 1 1 1 1 1; -;49 10802 11154 48 8536 29475 15024 25450 50 14651 46 29475 5241 2961 17336 0 15024 1717 2115 20215 1 9216 2726 1 69 1 2950 5;19 20 20 20 21 21 21 0 0 0 0 21 0 0 0 0 21 0 0 0 0 0 0 0 0 0 1 2;25 26 26 26 27 27 27 33 0 33 0 27 33 33 33 0 27 33 34 33 0 33 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;328 255 2 328 5;0 0 0 1 2;13 33 0 1 2;1 1 1 1 1; -;1900 2159 259 6063 2399 1916 71874 46 1619 12 8617 2959 2783 36 1900 2159 259 36 6063 2399 1 1352 1 114 119 1 114 5;0 0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 0 17 0 1 2 0 1 2;33 34 34 31 32 33 33 0 0 0 33 33 33 0 33 34 34 0 31 32 0 33 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10182 58 56 227 3 94 70 30 3948 178 897 12 60 1 4 9;17 18 0 0 0 0 0 0 9 10 0 0 0 0 1 2;1 2 0 0 0 33 0 0 33 34 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;50 55 6948 126 1550 12611 0 10497 70 75338 479 2471 1 628 2762 257;0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18;0 0 33 0 0 33 0 33 0 33 0 0 0 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14806 15800 27156 18391 2920 129 11711 20175 1263 735 1 4 9;0 0 0 0 0 0 0 0 0 0 0 1 2;11 12 33 0 33 34 21 22 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;9992 78 602 2179 0 378 7 548 49 12528 21354 48 7038 602 0 2 0 6521 81 78 131;0 0 0 0 0 0 0 0 45 46 46 46 0 0 0 0 0 0 0 0 0;0 21 22 33 0 21 22 33 53 54 54 54 33 33 0 0 0 21 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;459 106 29 97 74 93 32 17305 1214 118 5337 848 24 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 35 36 36 0 0 0 33 33 0 33 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13856 106 5243 7 780 13 181 54 5680 7 1247 54 346 4861 7 780 40109 238 11816 7 214 1334 603 10 20 156 17 13 29 893 992 166 22 165 3 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 0 33 34 0 0 0 0 0 0 0 0 0 33 0 0 0 33 34 34 33 34 33 0 33 34 34 0 33 34 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;640 3565 8865;0 0 0;33 34 33;1 1 1; -;2175 450 11338 122 528 21804 3 593 14839 8 10 20 501 8 15 613 2685 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;33 0 33 0 0 33 0 0 0 0 33 34 0 0 0 33 33 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7195 549 249 22 11 55379 3 99 59 1 4 9;0 0 0 0 0 9 0 0 0 0 1 2;33 0 33 34 0 13 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;1565 459 524 106 29 97 21 155 746 331 3847 15 143 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 35 36 36 0 33 34 0 0 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1539 5 1109 131 46 45 467 3119 94 136 205 1286 0 67 2061 8422 3261;1 2 0 0 0 17 18 0 0 0 0 0 0 0 17 0 0;1 2 33 33 0 33 34 33 33 34 34 33 0 0 31 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;20 478 12239 51696 17 12239 67372 8112 17 11 7865 59 59 59 59 59 59 59 59 1 706 2081 7412 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 33 0 33 33 34 0 0 33 0 0 0 0 0 0 0 0 0 27 28 28 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;151 244 11781 563 1627 136 3 2420 141 1 615 1 210 1 141 1 3956 615 5;0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 43 44 33 0 0 33 34 0 33 0 33 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;773 4783 167 5340 33 12 33 1 773 4783 167 5340 33 12 33 277 14 31 167 211 16;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 33 33 34 34 0 33 0 33 33 33 34 34 0 0 33 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1629 19580 3 747 770 1 4 9;0 0 0 0 0 0 1 2;33 34 0 33 34 0 1 2;1 1 1 1 1 1 1 1; -;92 5910 17470 37 1 80 5910 1 1177 3515;9 0 0 0 0 0 0 0 0 0;13 33 33 33 0 0 33 0 33 34;1 1 1 1 1 1 1 1 1 1; -;238 68 3003 2473 30 1431 544 2547 11 1451 12 453 17 62 284 2335 1146 74 1060 9819 74 6429 2020 106 2547 1021 66 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 33 0 33 34 33 33 34 33 34 0 0 0 33 34 0 33 33 0 33 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1131 3226 20152 1074 0 2 0 3384 424 687 535;0 0 0 0 0 0 0 1 0 0 0;33 34 34 34 0 0 0 1 33 34 0;1 1 1 1 1 1 1 1 1 1 1; -;636 2 73 5089 2 315 2 35 19 2 41 72;0 0 33 34 0 0 0 0 0 0 1 2;33 0 55 56 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 10059 1036 241 527 973 146 254 12090 1 5829 15199 2 146 100 1 110 54 115 100;0 0 0 9 0 0 0 29 0 0 29 0 7 8 8 8 0 0 0 0 0 0;0 0 0 13 0 0 0 0 0 0 43 0 11 12 12 12 33 0 9 10 10 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5557 6373 6975 3900 87 1 5557 6373 6975 3900 87 352 259 1 5557 6373 6975 3900 423 87 104 269 0 2 0 318 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 33 0 31 0 33 33 33 0 31 33 34 0 33 33 33 0 33 31 33 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;103378;0;0;1; -;1373 59 4280 56 70 1444 238 29 4878 13 332 363 24 17 1 174 601 799 117 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 46 46 46 0 0 1 2;33 0 31 33 34 0 33 34 21 33 34 34 0 0 0 53 54 54 54 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;126 433 703 3 755 3340 54 1403 946 1395 12 21 8 1098 1389 134 11607 1389 8 20 676 8 1727 3382 13 124 17 17 1 4 9;0 17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 0 0 33 33 34 33 0 33 34 0 0 33 0 0 33 0 33 34 0 0 33 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;545 63 11807 2939 91 180 1 4 9;9 0 9 0 0 0 0 1 2;13 0 13 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;768 555 348 12 64 6471 22 89 11 33 354 48011 1047 8 73 157794 1017 8 524 212 11451 7 7 7 0 2 0 45391 1875 0 2 0 3237 7 2196 0 45391 0 2563 10847 205 197 0 2 0 3237 7 2196 0 45391 197;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 34 33 34 34 34 33 34 0 0 0 0 0 0 33 0 0 0 0 33 0 0 0 0 0 0 33 33 0 0 0 0 0 33 0 33 0 33 33 33 33 0 0 0 0 0 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2837 2124 2060 423 1 2837 2124 2060 423 37 1 410 2837 2124 2060 423 104 26 182 0 2 0 111;11 12 12 0 0 11 12 12 0 0 0 0 11 12 12 0 0 0 0 0 0 0 17;15 16 16 33 0 15 16 16 33 34 0 33 15 16 16 33 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13008 75 376 533 73 381 1036 277 0;9 10 0 0 0 0 0 0 0;13 14 33 34 33 34 0 0 0;1 1 1 1 1 1 1 1 1; -;140 33917 15666 64 20 242 33 17 1 4 9;0 0 0 0 0 0 0 0 0 1 2;33 33 33 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;34 26859 0 2 0 277;9 9 0 0 0 0;13 13 0 0 0 0;1 1 1 1 1 1; -;9341 12 1584 2 588 648 5 0;0 0 0 0 1 2 2 0;33 33 34 0 1 2 2 0;1 1 1 1 1 1 1 1; -;536 3448 5299 1780 267 28 18;17 18 18 18 18 18 18;13 0 0 29 30 33 34;1 1 1 1 1 1 1; -;14390 2727 611 1886 164 3 48690 10 20 657 3 60 1 4 9;0 23 24 24 0 0 0 0 0 0 0 0 0 1 2;33 29 30 30 0 0 33 33 34 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12 105818 102 38 146 97 8 8065 2859 306 34629 448 1 297 100 1 316 100 1 316 197;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 17 18;0 0 33 34 33 34 0 33 0 0 33 33 0 33 34 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1471 1073 1517 684 230 168 5 46 105 38 1073 1517 684 230 587 171 2 168 171 2 3055 1020 2 1075 196 2 1448 5042 592 2 2146 361;0 9 39 40 40 40 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 0;0 13 47 48 48 48 0 0 33 34 13 33 34 34 33 33 0 33 33 0 33 33 0 33 33 0 33 34 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3527 9089 649 6883 394 1896 28 18 1 14 443 71 1 1489 71 1 908 71 1 434 71 1 123 147 1 116 71 16 196 2 1841 510;17 18 18 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 35 36 13 14 33 33 34 0 0 33 34 0 33 33 0 33 34 0 33 33 0 33 33 0 33 34 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;131283 643 2180 1 4 130 112;0 0 0 0 1 2 2;0 0 33 0 1 2 2;1 1 1 1 1 1 1; -;428 1718 0 9205 0 2041 20 665 6153 1 4 9;5 6 0 0 0 0 0 0 0 0 1 2;33 34 0 33 0 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;8532 80806 1564 10 20 156 17 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0;33 0 0 33 34 33 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1; -;1452 1165 1612 885 1 1452 1165 1612 885 37 1 1452 1165 1612 885 104 26 182 0 2 0 111;23 24 0 0 0 23 24 0 0 0 0 23 24 0 0 0 0 0 0 0 0 17;29 30 33 0 0 29 30 33 0 33 0 29 30 33 0 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;80 383 3782 2 281 421 1368 383 503249;0 0 0 0 1 2 2 2 0;0 33 33 0 1 2 2 2 0;1 1 1 1 1 1 1 1 1; -;34 41 50 492 58 39 4832 585 1 1031 1 35 2164 1486 31 95 1 279 19;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0;33 34 34 34 34 0 33 0 0 33 0 33 21 22 33 34 0 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1684 113 1269 345 32 452 1547 864;17 18 18 18 18 18 18 18;31 32 32 32 32 32 32 32;1 1 1 1 1 1 1 1; -;2980 36736 74 6609 36736 464 17703 426 15110 1405 901;0 0 0 0 0 0 0 0 0 0 0;33 0 0 33 0 21 22 0 29 33 33;1 1 1 1 1 1 1 1 1 1 1; -;13796 148 1554 6680 6118 1609 270 2094 58 2 10772 1632 369 577 1720;9 10 0 0 0 0 0 0 0 0 9 10 17 18 18;13 14 33 33 33 33 33 33 0 0 13 14 31 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1290 162 38 8 663 115 30 28981 1935 3 7497 2547 4951 3101 3919 8 11305 38 663 115 1017 30 208 1112 238 29 839 5253 3038 289 28981 5253 3 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 33 34 0 33 34 0 33 33 33 33 33 0 33 34 33 34 34 0 33 0 33 34 0 33 33 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9059 526 854 52 6740 8961 15 587 1520 776 1 9059 526 854 52 1 78 1074;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 22 22 33 34 0 33 33 33 0 21 22 22 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5627 40053 12206 0 2 0 5627 40053 12206 534 36 887 55 0 2 0 4480 912;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6;39 33 33 0 0 0 39 33 33 33 0 33 34 0 0 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6701 178 7592 249 1328 1891 25074 101 3006 2872 152 4461 13 799 13 2469 1 188 69 108;0 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 33 33 31 0 33 33 0 0 0 0 0 0 0 9 10 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;507080 10 20 156 1 507080 3 300 17845 1 1526 1 6119 731 1 140 7401;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 34 33 0 0 0 33 33 0 33 0 33 34 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5716 0 2 0 80 30 1682 0 920 0 103 9130 0 15141 3639 3 639 0 2 0 11916 11507;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 0 0 0 0 0 33 34 34 0 0 0 0 33 33 0 33 0 0 0 33 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;23648 573 116 8846 138 230 1 88 1089 116;0 0 0 0 0 0 0 9 0 0;33 34 33 33 34 34 0 13 31 32;1 1 1 1 1 1 1 1 1 1; -;683 28750 167 446 1 683 28750 167 446 261 0 2 0 2818 434 6575 0 3082 8812 2828;9 17 18 0 0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 0;13 31 32 33 0 13 31 32 33 33 0 0 0 0 33 33 0 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1757 14569 0 2 0 3094 253;0 0 0 0 0 1 2;25 26 0 0 0 1 2;1 1 1 1 1 1 1; -;420292 1836 12238 100 1647 14 5760 875 14036 50 791 90 3157 227 24 17 16;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 33 33 33 0 21 0 31 0 33 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2241 4216 48403 2525 2177 652 346 8 51243 7137 1442 9545 10104 353 59 59 90 3010 350 4312 15 124 21 17 17;0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 27 0 33 34 0 0 0 35 36 33 0 0 0 0 33 34 0 33 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;627 45 3 13053 969 32 27361 969 10 30 453 13 29 392 45 24 60 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 21 33 0 33 33 0 0 33 34 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13091 27 2356 107 1 123 1 707 1 147 1 469 1 648 2 4615 5;17 18 18 18 0 0 0 0 0 0 0 0 0 0 0 1 2;13 14 21 22 0 33 0 33 0 33 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4151 2710 711 639 68 0 105 1007 4151 2710 711 44 201 2 1392 2722 2 191 4299 5;0 23 24 0 0 0 0 0 0 23 24 0 0 0 0 0 0 1 2 2;33 29 30 33 34 0 33 33 33 29 30 33 33 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;90 597 12 2414 22 1777 3 47;0 0 0 0 0 0 0 0;0 0 0 33 0 0 0 1;1 1 1 1 1 1 1 1; -;11599 2258 438 0 26 0 4030 456 1 157 27 11599 2258 456 138 4294 28 18;17 18 0 0 0 0 0 0 0 17 18 18 18 18 18 18 18 18;0 0 33 0 0 0 21 22 0 31 32 32 32 32 32 32 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5911 237 40191 573 22517 74 26702 8224 59 102 38 2329 116 58 12305 1106 45 7 7 7 7 2 247 142;17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0;31 32 0 33 34 0 0 33 0 33 34 33 33 34 33 33 34 0 0 0 0 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;856 1942 1452 141 75 6 2081 622 31 205 170 95 6 2464 331 5582 3323 6 2729 9653 94 31 6 94 31 6 2464 3598 21302 19 291;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 25 33 34 0 0 0 0 21 22 33 33 0 0 33 34 0 0 33 33 33 33 0 33 33 0 0 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;320 320 48405 1192 2260 3 22 320 320 214 1334 579 10 215 574 492 164 3 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 33 34 0 0 0 0 0 0 33 0 0 21 22 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7486 2022 1333 8 420 27456 11 64 442 1 26496 6843 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 0 33 33 33 34 0 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;647 174 1706 2361 67 296 1032 194 1 5701 194 294 343 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 33 0 33 33 34 0 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;41706 26559 221 6 1275 1570 14228 3 2979 1240 2974 600 20 60 0 2 0 612 295 42;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 33 0 0 0 33 33 0 33 33 33 0 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;470 19774 62 424 66;21 22 0 0 0;27 28 0 33 0;1 1 1 1 1; -;33148 3 208 3919 2 2 49 45 12820 48 7211 38 2077 276 0;29 0 0 0 0 0 27 28 28 28 0 0 0 0 0;37 0 33 33 0 0 0 35 36 0 33 34 21 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;459 146 522 8 7060 362 12948 4194 600 2389 1862 1 1422 217 415 5 1 1572 7 65;0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2 0 0 0 0;33 0 0 0 33 33 33 0 0 0 0 0 1 2 2 2 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;96316 729 636 6 96316 1791 636 6 96316 137 77 2579 3 636 201 40 0 2 0 15292 202 5;21 0 0 0 21 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0;27 33 33 0 27 33 33 0 27 21 0 33 0 33 33 33 0 0 0 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1266 1311 46 64 346 249 731 3 13184 3352 5987 0 448 2421 0 2 0 1311 1545;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 0 0 0 33 34 33 33 33 0 33 33 0 0 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;40213 17008 391 2036 424 1799 838 62 11 1089 349 3618 66 2772 1196 170 40 1 877 51 2210 23 44 2013 51 1942561 25 2 916 2093 163 7 702 2759 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0;33 33 1 2 33 34 33 0 0 33 34 33 0 33 33 33 33 0 33 0 33 0 33 33 0 0 0 0 1 2 33 0 1 31 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2230 3289 1571 0 3378 6723 5820 1571 0 5076 390 6723 1575 13040 0 7086 425 5648 13040 16 37 6 87 6 44 6 3289 1571 6 396060 2 395 5;0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 1 2;0 33 0 0 0 33 33 33 34 0 0 0 33 33 33 0 33 0 0 33 0 33 0 31 0 33 0 0 0 0 27 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;24529 5136 0 35 1288 1 14936 1288 19 1 2 0 412 1848 58 0 16871 55 0 431 26 471 26 1705 1 8410 1288 202 19 201;0 0 0 0 0 0 5 6 6 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 0 33 33 0 9 10 10 0 0 0 13 47 48 0 0 0 0 33 0 33 0 33 0 33 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;105 1900 4171 601 306 8036 298 346 34690 3 286 9753 59;0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 34 33 34 33 33 34 34 0 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;3133 30 101 56 15737 13 806 3739 0 2 0 4118 253 0;0 0 0 0 0 0 0 0 0 0 0 5 6 0;33 33 34 34 33 0 21 22 0 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2280 5845 653 4124 5845 0 1979 2538 0 35062 450 7615 3682 839 252 5845 16 37 6 87 6 44 6 5845 26 495 2366 6 389 771 56 420 267 28 18 2 395 5;0 9 0 0 0 0 0 0 0 0 0 23 24 24 24 24 24 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 18 0 1 2;0 13 33 33 33 33 0 33 33 0 33 29 30 30 30 30 30 0 33 0 31 0 33 0 33 0 33 33 0 31 32 32 32 32 32 32 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;143 1175 751 5023 1621 1308 20 436 84 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 34 0 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;46747 608 0 960 8627 1 405 1 35 19 31 95;0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 0 33 0 1 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1; -;2092 518 120253 252 25001 5763 1078 1 2092 518 120253 252 25001 5763 1078 37 1 2092 518 120253 252 25001 5763 1078 104 26 182 0 2 0 111;0 0 0 0 23 24 24 0 0 0 0 0 23 24 24 0 0 0 0 0 0 23 24 24 0 0 0 0 0 0 17;33 0 0 0 29 30 30 0 33 0 0 0 29 30 30 33 0 33 0 0 0 29 30 30 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;420 159 602 161 808 435 4614 0 2 0 364 5;0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 34 33 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;849 101 725 7529 475 8 28777 6897 2017 3 20170 17 1 68 136 106 429 1 106 429 100;7 8 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0;11 12 0 33 34 0 0 13 0 0 0 0 0 0 0 33 34 0 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;17167 27 48812 6069 28 18 0 277 0;17 18 18 18 18 18 0 0 0;13 14 31 33 33 34 0 0 0;1 1 1 1 1 1 1 1 1; -;25782 67 190 6277 11 12385 59 49 3869 5688 129 48 900 278 33525 1 3333 5 0 9781 8059 7 1054;21 0 0 0 0 0 0 45 46 46 46 46 0 0 0 0 1 2 0 0 0 0 0;27 0 21 22 0 33 0 53 54 54 54 54 33 34 0 0 1 2 0 27 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;460 26230 611 5 3 1665 5 853 10 20 1 4 9;0 0 0 0 0 0 0 0 0 0 0 1 2;33 21 22 22 0 33 34 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1476 31932 3209 1315 11439 8 1933 7544 216 21 1415 8 134 89 11 50 2469 8 922 2408 21 8 3209 11 0 42 8 166 22 302 12 1681 8 15 143 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 33 0 33 0 0 33 33 34 33 0 0 33 34 0 0 0 33 34 0 0 33 0 0 0 0 0 0 0 33 34 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11 7973 12 59991 59 3 3591 0 2 0 3184 2782;0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;900 36056 80 669 3165 890 17 1 18900 158 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 33 34 0 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 19320 11093 12 2676 48 3828 534 40 170 40 2 3828 301 40 5;27 28 28 28 28 28 0 0 0 0 0 0 0 0 0 0;0 35 36 36 36 0 33 33 33 33 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;655 103 22 216 11145 1068 32 404 41534 3 3696 1 2122 2641 100 1 28374 78;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18;33 0 33 34 33 34 0 0 33 0 33 0 21 22 33 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 34 25095 22001 1187 208 58 0 34 25095 22001 1187 208 58 22001 1187 6747 16 1688 400 1 37 1 44 1 3151 1 4494 1 23 25095 25 0 2 870 351 632 400 1688 5;0 0 0 0 0 0 0 0 0 0 27 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;0 33 34 33 34 33 0 0 33 34 33 34 33 0 33 34 0 0 33 34 0 33 0 33 0 33 0 33 0 0 33 0 0 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1209 67 1574 159 257 988 1159 59;0 0 17 18 18 0 0 0;33 34 34 33 33 33 0 0;1 1 1 1 1 1 1 1; -;98 174 452 2223 56636 349 954 0 2 0 364 5;0 0 0 0 0 0 0 0 0 0 1 2;21 22 22 22 33 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;963 22 30 1804 178 164 199 21 13 29 1049 8 178 164 3 150 532 6367 22 17052 1203 96 9881 22 589 8 22 89 11 589 8 4312 659 32395 21 8 41974 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;55 56 0 33 0 0 0 33 34 0 33 0 0 0 0 33 34 33 0 0 33 0 0 21 22 0 33 34 34 0 0 33 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;284 57 430 378 45 800 278 146 29 1431 549 8 853 800 278 3 549 45 12 1584 1112 29 1431 549 10 2367 13 29 1431 549 3 52 2471 8 1361 13 1255 11 162 422 800 329 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 33 33 34 0 0 0 33 34 0 33 0 0 0 33 34 33 34 0 0 33 34 0 33 34 34 33 34 0 0 0 0 0 33 34 0 33 34 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2844 27 67 34 1896 98 10613 2066 1148 1602 1 680 69 5;9 10 0 17 18 18 0 27 28 28 0 1 2 2;13 14 0 31 32 32 33 35 36 36 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;200 386 3 79 39 1312 3847 74 824 8 39 127 173 369 15629 8 39 4780 455 1369 1117 8 12 236 3397 8 89 1190 542 15 143 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 37 38 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;0 0 0 21 33 34 0 0 0 0 33 34 34 45 46 0 33 34 0 33 0 0 33 34 33 0 0 33 33 34 34 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1942561 3 47;0 0 0;0 0 1;1 1 1; -;6151 1311 0 2 16732 98 8036 20521 407 6548 5030;0 0 0 0 17 18 0 0 0 0 0;33 33 0 0 31 32 33 33 0 47 33;1 1 1 1 1 1 1 1 1 1 1; -;57130 31104 1 4711 268;17 18 0 17 0;31 32 0 33 21;1 1 1 1 1; -;3589 44662 19528 3 661 6 3589 44662 19528 15 109 1721 6 3589 44662 19528 3 1937 661 1 5719 94069 1 33 2548 5;17 11 12 0 0 0 17 11 12 0 0 0 0 17 11 12 0 0 0 0 0 0 0 1 2 2;15 15 16 0 33 0 15 15 16 33 34 33 0 15 15 16 0 33 33 0 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;141 30343 666 959 51 796 30343 666 959 12 5128 9552 796 0 171 11 21426 6 9536 1463 89 11 7 14979 121 4669 959 6 3026 30343 666 959 7 932 4970 1 19 31 95 0 2 0 1249 7 65;0 19 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 1 2 0 0 0 0 1 2 2;33 25 26 26 0 0 25 26 26 33 34 33 0 0 33 0 33 0 33 0 33 34 0 33 0 33 0 0 33 25 26 26 0 0 27 0 1 2 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;98445 0 91501 0 36 0 9628 0 25044 0 36 0 31195 0 16034 0 556 0 31380;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;27 0 21 0 0 0 31 0 33 0 0 0 21 0 0 0 33 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6361 38514 1686 26403 33 24 17 1 4 9;0 0 0 0 0 0 0 0 1 2;0 33 33 33 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;29923 3160 1367 3453 3404 13841 134 10 562 215 324 242 56 3 6 9 3 165 103 1 4 9;17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;31 33 33 34 33 33 0 0 0 0 0 0 0 0 0 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;102 38 26 26 92 1589 616 28 18 102 38 92 1589 14654 211 71 263 372 32 4282 150 833 127 393 481 1 271 6231;0 0 0 0 17 18 18 18 18 0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 0 0 31 32 32 32 32 33 34 31 32 51 33 33 34 33 0 33 33 34 21 22 33 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;160 21904 145 112 1228 227 1613 1187 64 1280 223 5590 964 93 227 24 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 0 33 0 21 0 0 33 34 33 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5172 27 52826 75 662 153 1 52826 75 69 2 8598 134 5;17 18 18 18 18 18 0 9 10 0 0 1 2 2;13 14 31 32 32 32 0 13 14 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14970 30 2114 275 6587 77 179 0 634 38152 8492 118 236 5411 34 2 1353 99;0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 1 2;33 0 0 0 33 33 34 0 33 33 34 34 0 0 13 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1566 1224 34 46 269 42762 9104 7403 0 13454 121 488 1 294 2085 1 114 268 1 114 5;17 18 18 0 0 0 17 18 0 0 0 0 0 0 0 0 1 2 0 1 2;31 32 32 0 33 0 31 32 0 33 33 34 0 33 33 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1227 4127 8 3201 39 13 3254 1227 3266 2626 128 84 729 41462 3254 1227 3266 3 229 8 134 93 30 3278 1819 361 84;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 33 33 34 0 33 0 33 33 0 33 0 0 33 0 0 33 0 0 0 33 34 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;162 246 52488 3 424 687 0 2 0 4118 253 0;0 0 0 0 0 0 0 0 0 5 6 0;33 34 33 0 33 34 0 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1; -;1683 1505 10 4342 48782 311 3 603 8 21617 2017 10 12 10 135 10 7979 4888 21 1 3087 54 2450 85 43 1 4 47;9 10 0 0 0 0 0 0 0 21 22 0 0 0 0 0 0 0 0 0 45 46 46 46 0 0 1 2;13 14 0 33 33 0 0 0 0 29 0 33 34 34 0 0 33 33 0 0 53 54 54 54 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;443567 0 209296 0 45496 51 0 239 0 27816 0 33840 0 1797 0 91168 6 0 114447 0 513 0 76634 0 268971 0 3773 0 94300 0 474 0 4513;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 0 33 0 0 0 0 33 0 33 0 33 0 33 0 0 0 0 33 0 55 0 0 0 33 0 0 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 49 7008 58840 3 10921 48 309 90 30 214 1005 7685 718 63 3 33 2180 13663 2624 6 43315 1970 285 45 4712 7 16 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 0 33 34 0 13 0 0 0 0 0 0 33 33 0 0 0 33 33 33 0 0 0 33 0 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;459 511 29 97 43008 18739 2254 85796 10 12 10 2969 8 15 70 7413 4683 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 0 33 34 34 0 33 34 34 33 0 33 34 33 33 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 32476 1942561 16 32476 26 1597 45 112 16837 9851 24115 6713 1512 866 26 26 593 520 26 1942561 0 32476 23 220371 0 1942561 25 1979 2737;0 17 0 0 17 0 0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 17 0 17 0 0 0 0 0;0 31 0 0 31 0 33 0 0 33 0 33 33 29 30 0 0 33 34 0 0 0 31 0 31 0 0 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2477 1931 376 3685 7591 126 248 145 134 12 2586 1 4 9;17 18 18 18 0 0 0 0 0 0 0 0 1 2;13 14 14 33 34 0 0 0 0 55 56 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6783 29696 32149 2445 15 124 60 29696 32149 2445 2054 24 60 1 564 2445 1 280 382 1 280 463;9 9 9 0 0 0 0 9 9 0 0 0 0 0 9 10 0 0 0 0 1 2;13 13 21 33 0 0 0 13 21 33 33 0 0 0 13 14 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;487 2966 134 699 56 13 29 11589 12 333 176 3 42 0 2 0 78068 2817 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 0 33 34 33 34 34 34 0 0 0 0 0 33 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 16463 42829 1067 1339 1252 307 16 1 16463 42829 1067 1339 1252 307 37 1 16463 42829 1067 1339 1252 307 104 1 16463 42829 1067 1339 1252 307 87 1 395 5 0;0 17 7 8 0 0 0 0 0 17 7 8 0 0 0 0 0 17 7 8 0 0 0 0 0 17 7 8 0 0 0 0 0 1 2 0;0 11 11 12 33 0 33 0 0 11 11 12 33 0 33 33 0 11 11 12 33 0 33 33 0 11 11 12 33 0 33 31 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9212 33751 397 3 1647 2116 0 23687 45492 261 81 1647 0 50105 1647 0 181 425 2181 0 38550 2181 0 2181 2181 3407 0 1876 21703 0 9212 584 0 18076 1125 8221 2634 0 13 29 584 0 55261 0 4577 173 49086;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0;33 33 33 34 33 33 0 33 0 33 0 33 0 0 33 0 33 34 0 0 33 0 0 0 0 0 0 33 39 0 33 0 0 1 2 33 33 0 33 34 0 0 0 0 27 28 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;78642 22896 29899 45 11 187 2687 194 3 1807 3574 24 17 0 2 0 18389 67 71 1374 55 0 2 11389 100;0 13 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0;29 17 33 33 34 0 33 34 0 33 33 0 0 0 0 0 33 0 33 33 0 0 0 13 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;72850 202 58 687 633 2 72850 202 58 1032 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 22 33 34 0 21 22 22 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1179 29164 13617 578 1 1179 1 316 100 1 316 197;9 10 0 0 0 9 0 1 2 0 1 2;13 14 33 33 0 13 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;2833 42 38 1403 2481 32 1539 42 38 1403 2481 11 20 454 8 10 20 613 432 17 2 17892 87511 5 197;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0;33 34 33 34 34 0 33 34 33 34 34 33 34 33 0 33 34 33 0 0 0 1 2 2 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 1306 175 11 1202 11 28293 104 17 1 4 9;9 10 10 0 0 0 0 0 0 0 1 2;13 14 14 33 34 34 29 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;380 599 3779 192 662 28 18 6 488 1475 51 0 104 6 4078 62 1611 344 653 66 46 728 1165;17 18 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 0 33 34 33 34 0 0 0 0 0 33 0 33 0 0 21 22 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15 30 34 1838 153 261 56 510 16166 18025 17 1 4 9;0 0 17 18 18 0 0 0 0 0 0 0 1 2;0 21 22 33 34 33 0 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;30 38213 16638 1855 469 39 13 393 216 9252 769 158 53 53 1945 235 78 1350;0 0 0 0 0 0 0 0 21 22 45 46 0 0 0 0 0 0;0 33 33 34 33 33 34 34 27 28 33 34 33 34 1 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8333 1589 98 647 174 1706 1807 108;17 18 18 0 0 0 0 0;31 32 32 33 34 34 34 33;1 1 1 1 1 1 1 1; -;35714 102 38 4573 583 13 324 1 50 5077 5 1 279 5;17 0 0 0 0 0 0 0 1 2 2 0 1 2;31 33 34 33 0 33 34 0 1 2 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 4793 4491 1813 2337 16 1 4793 4491 1813 2337 44 1 37 0 2 0 322 5;0 0 11 0 0 0 0 0 11 0 0 0 0 0 0 0 0 1 2;0 33 33 0 33 0 0 33 33 0 33 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;39 479 248 807 0 335 1262 264 1025 2888 930 10061 4349 39 479 1134 0 478 10410 20537 1 11783 24324 905;0 0 0 0 0 21 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 0 0 0 27 28 33 34 0 0 0 0 33 34 0 0 0 33 33 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1038 39 628 8181 643 17617 20862 414 1299 2741 8181 2 45 81 5;0 0 0 0 0 0 17 18 0 0 0 0 1 2 2;21 22 22 22 22 33 31 32 33 21 22 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2715 563 2232 397 4583 572 2105 8 15418 392 688 84 84 84 84 84 1 29864 42 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 1 2;33 34 34 33 0 33 33 0 33 0 0 0 0 0 0 0 0 13 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;151 156007 377 213 6 96 122 3193 3 10236 523 320 1082 5839 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 21 45 46 0 0 0 0 0 0 0 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;397 3 8382 10 12 10 64 16641 284 14072 164 2743 12 278 675 212 203 17 0 2 0 5361 110 0 2 0 5361 197;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 0 0 0 0 0;33 34 33 33 34 34 0 33 0 33 0 0 33 34 0 33 34 0 0 0 0 33 33 0 0 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;36937 76 44 6 35 6396 1 7260 12431 926;0 0 0 0 0 0 0 0 0 0;29 30 33 0 33 0 0 33 33 33;1 1 1 1 1 1 1 1 1 1; -;1772 62197 3 47;0 0 0 0;0 0 0 1;1 1 1 1; -;742 59 59 0 3073 1168 722 11 89 11 4191 567 52 99 213 203 17 1913 219 1639 716 572 1568 17 1892 15 589 17 0 742 203 84 151 99 135 341 160 21 84 1 4 9 0;0 0 0 0 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;0 0 0 0 13 14 33 33 34 34 33 34 0 0 0 0 0 33 0 33 0 33 0 0 33 33 34 0 0 0 0 0 33 34 0 0 35 36 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;191828 0 2 0 412 9449 1708 52215 786 565;21 0 0 0 17 18 18 18 18 18;31 0 0 0 21 22 22 31 33 0;1 1 1 1 1 1 1 1 1 1; -;196 237 183 4303 0 12 3727 1248 21 4142 63 10881 0 58 369 63 215 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 33 0 33 34 0 0 33 0 33 0 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;42260 249 100676 54863 0 6 6403 609 1 4 9;29 0 0 0 0 0 0 0 0 1 2;37 0 33 33 0 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;82 200 5796 83 82 68652 83 661 333 30 6049 2286 36 3417 1892 36 103 5796 1 328 255;0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0;0 0 0 0 0 33 0 33 0 0 33 33 0 33 34 0 0 0 0 13 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6662 6658;0 0;33 33;1 1; -;49 5576 3888 62 7163 55 66 48 14 2006 0 1733 0 2362 16 2 0 553 941;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 45 46 0 33 34 0 0 0 33 0 33 0 33 0 0 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;142519 2 0 156376 479 29 73 22735 0 90 2081 2219 0 50 68186 2 167 2 35 19 2 41 72;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;27 0 0 13 33 34 34 34 0 0 33 34 0 33 34 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10629 619 249 634 240 1957 21 59 12 944 90 212 13557 1 17914 505 1 114 204;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 0 33 0 0 0 0 33 34 33 34 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;923 16026 6447 0 2 0 277;9 0 0 0 0 0 0;13 33 0 0 0 0 0;1 1 1 1 1 1 1; -;384 106 304 8339 3145 3890 0 740 224 0;0 0 0 39 0 0 0 0 0 0;33 34 34 47 33 33 0 33 0 0;1 1 1 1 1 1 1 1 1 1; -;19645 2086 0 2 0 188 271 202;0 0 0 0 0 5 6 6;55 0 0 0 0 33 34 34;1 1 1 1 1 1 1 1; -;7030 753 45 293 3658 1654 3 9718 84 1 4 209 132;37 0 0 0 0 0 0 0 0 0 17 23 24;33 0 33 34 0 33 0 33 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1; -;17433 5694 4709 28270 23 68 25 2 92 8939 2096 23 81 2425 178 25 2 92 2 588 648 5;0 29 30 30 0 0 0 0 9 10 0 0 9 10 0 0 0 9 0 1 2 2;33 37 38 38 0 0 0 0 25 26 33 0 33 34 0 0 0 13 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 321214 13326 5 1 4 61;9 0 0 0 0 1 2;13 0 33 34 0 1 2;1 1 1 1 1 1 1; -;7611 269 7849 0 7611 37 1503 248 38 81 179 1 841 268 1 114 268 1 114 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 1 2;33 34 33 0 21 22 0 33 34 33 34 0 33 21 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4853 985 349 2166 242 0 2 0 364 5;0 0 0 0 0 0 0 0 1 2;33 33 34 33 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;56 797 5116 835 574 15370 8714 3928 2 2 342 1006;0 0 0 0 7 8 0 0 0 0 0 0;0 0 0 33 11 12 33 33 0 0 35 36;1 1 1 1 1 1 1 1 1 1 1 1; -;22 346 3618 254 3 1233 634 84 84 84 84 62 2730 44510 66 1 2715 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 0 0 33 34 0 0 0 0 0 33 34 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1368 1086 23 2030 25 51 75281 1086 1 95749 12912 326 1359 1491 1 33 942 5 10199 12018;17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0;33 33 0 33 0 0 21 22 0 33 33 9 33 33 0 1 2 2 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;152804 130048 38862 784 1540 4145 2255 0 1279 7793 373 1621 815 2 247 401;0 0 21 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 27 33 34 33 33 0 0 33 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; diff --git a/PaddleNLP/unarchived/chinese_ner/data/train_files/train_part_1 b/PaddleNLP/unarchived/chinese_ner/data/train_files/train_part_1 deleted file mode 100644 index bbdd1deaab7cfdcdefd06f1cbbf5333cf0bef344..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/data/train_files/train_part_1 +++ /dev/null @@ -1,500 +0,0 @@ -;543 5634 257 2194 652 0;17 18 18 0 0 0;31 32 32 33 34 0;1 1 1 1 1 1; -;14 32 1681 13691 17010 16 2 2770 3207;0 9 10 10 10 0 0 9 0;0 13 14 14 14 0 0 13 21;1 1 1 1 1 1 1 1 1; -;1354 113 2424 1036 550 38 42 2177 544 153332 3399 343;9 10 0 0 0 0 0 0 0 0 0 0;13 14 33 0 33 34 33 34 34 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1; -;8715 67 9724 4753 0 5358 17933 11 20 64 1 8715 67 9724 1 3972 94 78 100 1 4538 7 3972 7 65;0 0 0 0 0 0 0 0 0 0 0 45 46 46 0 0 0 0 0 0 0 0 0 0 0;33 34 34 33 0 35 36 33 34 34 0 33 34 34 0 1 33 34 33 0 33 0 1 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;807660 0 105 0 1315209 0 73356 0 807660 0 5623 0 1315209 0 73356;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 0 33 0 0 0 33 0 21 0 21 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1142 13653 22741 1 52 2120 146 55779 987 1444 3420 252 1142 13653 22741 0 7288 728 5538 44464 17329 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 23 0 0 0 0 17;33 0 33 0 0 0 0 0 33 0 0 0 33 0 33 0 33 29 30 29 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;38862 32 60503 4515 3615 118 6408 3 597 597 1 5043 681 43 1 4 47;21 0 21 0 0 0 0 0 0 0 0 39 40 0 0 1 2;27 0 27 0 0 0 33 0 33 34 0 47 48 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1011 366 11 42 165 133 4750 12 583 7607 8 14294 12 136 3620 133 8 214 1334 603 20 156 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 0 0 33 0 0 0 0 33 0 0 0 0 0 33 34 34 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9294 46 9245 172 97104 2606 12 11158 24843 1 328 268;0 0 0 0 17 0 0 0 0 0 0 0;0 0 33 34 31 0 33 34 34 0 13 21;1 1 1 1 1 1 1 1 1 1 1 1; -;1978 2479 3 695 4514 78 76 348282 46 11822 2379 311 0 2 0 1767 0 2 0 861 3924;0 0 0 0 0 23 24 23 0 0 0 0 0 0 0 0 0 0 0 1 2;33 29 0 33 33 33 34 29 0 33 29 30 0 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6 2868 2600 143541 456 436 28 18;0 17 18 18 18 18 18 18;0 21 22 0 33 34 33 34;1 1 1 1 1 1 1 1; -;3461 275 3336 42245 13032 369 4031 46 20304 1585 1687 583 12 21 10371 1 36 20304 36 4630 75320 192 1044 1 328 5;21 22 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;27 28 28 27 33 33 34 0 33 33 33 0 33 34 0 0 0 33 0 33 21 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7339 1341 310 406 398 23 5455 25 2460;0 0 0 0 0 0 0 0 0;33 33 34 33 34 0 0 0 33;1 1 1 1 1 1 1 1 1; -;126 10031 510 3063 22 3 2290 775 60 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 33 34 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;136596 111979 4573 1404 8 811 136596 111979 4573 1404 3 123 1 147 1 811 469 0 2 0 460 43 6390 469;21 17 0 0 0 9 9 10 0 0 0 0 0 0 0 9 0 0 0 0 1 2 0 0;27 31 33 33 0 13 27 31 33 33 0 33 0 33 0 13 33 0 0 0 1 2 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7652 10550 8092 49514 68 74 8092 10 20 156 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 21 33 0 0 21 33 34 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;213 56 349 578 2 0 13 470 12 11158 18973 3 5503 578 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 33 0 0 33 34 33 34 34 0 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;54 298 11 254 2804 1851 8 397 524 200 29 97 21 8 93 194 210 24 17 1 4 79 9;0 0 0 0 37 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 21 22 33 33 0 33 0 0 0 0 0 0 0 21 22 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;629 642 33746 55 21963 0 8050 0 694 0 4841 1182 55 57 246 672 1904 15755 7262 59 53 31 291 53 387 5 8 19 35 31 95;0 0 0 0 33 34 34 34 34 34 34 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;33 33 34 0 55 56 56 56 56 56 56 33 0 0 0 33 0 33 0 0 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;384 106 304 745 468 6 541 112 3212 13 103 6 781 62 0 66 0 62 0 66 0 14882 0 62 0 66 0 62 0 66 138062 62 0 66 3 2859 1167 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;35 36 36 36 0 0 0 0 33 33 34 0 33 0 0 0 0 0 0 0 0 33 0 0 0 0 0 0 0 0 33 0 0 0 0 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;322 1491 51 46710 38689 0 4216 775 1 4962 1394 1 3017 4216 211 0 2 0 146 1228992 7 65 160 4216 5 0;1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;1 33 0 33 0 0 33 33 0 33 33 0 33 34 33 0 0 0 0 0 33 34 0 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;21118 62 22246 66 666 10091 4826 465 865 218 13363 1 188 1484;0 0 17 0 0 0 0 0 0 9 0 0 17 0;31 0 31 0 0 33 21 22 22 13 33 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;649 259 250 5270 746 8 1520 13 145 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 0 33 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;22 1200 700 278 4594 11 17899 15358 518 8 22 236 1133 19105 654 212 1854 8325 3 5276 620 8 12 1406 214 1143 22 24 17 1 4 209 132;0 0 0 0 0 0 37 0 0 0 0 0 0 39 40 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;0 0 33 0 0 0 45 33 0 0 33 34 0 47 48 0 33 0 0 33 34 0 33 34 0 33 0 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;902 29944 12987 1363 200 38 0 2272 750 3 13 421 10070 213 23 68 25 1 841 1 7842 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 22 33 33 34 0 33 33 0 33 34 33 34 0 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;32041 1919 32 6899 219 33 1 4 9;17 0 0 0 0 0 0 1 2;31 33 0 29 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;687 6840 3 3335 6269 5255 0 6 687 6840 3 25020 3014 2486 2761 1934 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 33 34 33 0 0 33 34 0 33 34 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;911 882 121 1166 286 1575 837 278 9424 14010 2 2 34 1590 5 0 1038 169 1590 3688 5145 0 835 77 50 300 2853 235 411;9 0 0 9 0 0 0 0 0 0 0 0 1 2 2 0 17 18 18 18 0 0 0 0 0 0 0 0 0;13 33 0 13 33 33 34 0 0 33 0 0 1 2 2 0 31 32 32 32 33 0 33 33 34 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1009 462 43 1648 26779 3 47;0 0 0 0 0 0 0;21 22 0 0 0 0 1;1 1 1 1 1 1 1; -;42741 762 10 893 3 4395 0 4435 3004 3 629 1581 78 0 1 0 3138 2269 0 3849 3758 7 65;0 0 0 19 20 20 0 0 0 0 0 0 0 0 0 0 17 18 0 0 0 0 0;33 33 34 25 26 26 0 33 29 0 33 33 34 0 0 0 31 32 0 21 27 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;339930 1131046 0 909256 0 7721 22801 10659 5506 0 200579 2722 0 13132 1 339930 1131046 866 1 6044 866 1 23763 12744 5 2 34 77 50 3 2 654 866 296 6260 2 65420 12744 394 197 2;17 18 0 0 0 0 0 0 0 0 43 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 9 0 0 0 0 23 24 0 0 0 0 0 0 0 0;31 32 0 13 0 33 33 33 0 0 51 33 0 33 0 31 32 0 0 33 0 0 1 2 2 0 13 33 34 0 0 33 34 33 33 0 29 33 13 14 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;958 721 125663 70912 91 1 4 9;9 0 0 0 0 0 1 2;13 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1; -;536 11697 5615 116 0 2 0 536 5000 3610 3224 28 18 0 2 0 1774 884 224 23 801 34 33 18 25;9 0 0 0 0 0 0 17 18 18 18 18 18 0 0 0 1 2 2 0 0 9 0 0 0;13 33 33 33 0 0 0 21 22 22 22 22 22 0 0 0 1 2 2 0 21 22 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1425 16 98436 406 10460 0 2 0 364 5;0 0 0 0 0 0 0 0 0 1 2;0 33 0 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;854 1528 151 15 1082 12 59068 43656 8 13 29 42 120 1791 460 1 4 9;45 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;53 54 54 0 33 0 33 33 0 33 34 34 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1844 6703 3176 2328 3656 5165 8284 265 57 6 2328 177 57 265 57 26 52 2328 177 57 7 222 23 57 25 2328 177 57 2 52 26 2328 2 52 3 673 23 52 25 6703 3176 2328 3656 3 599 1902 1504 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 33 0 33 0 33 33 0 0 0 33 0 0 0 33 34 34 33 0 0 0 0 0 0 0 33 0 0 0 0 0 33 0 0 0 0 0 0 0 33 0 33 0 0 21 22 22 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;328 5 678 131 3558 481 176 6349 1 1130 131 1 328 5;1 2 0 0 0 0 0 0 0 0 0 0 1 2;1 2 33 33 33 33 0 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3040 10051 254 4742 248 12 4038 1 4 61;0 0 0 0 0 0 0 0 1 2;33 33 34 34 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;191 816 18 138 23074 5258 311 13713 3 2113 18289 2 7762 5 0;0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;33 34 34 33 33 21 22 0 0 33 21 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;146258 46 1057 38 12 1570 795 2118 26116 2 2 34 1590 5 0 1038 169 1590 3688 5145 0 835 77 50 300 2853 235 411;21 0 0 0 0 0 0 0 0 0 0 1 2 2 0 17 18 18 18 0 0 0 0 0 0 0 0 0;27 0 33 34 33 34 33 33 33 0 0 1 2 2 0 31 32 32 32 33 0 33 33 34 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;218 46 1408 18122 12 1357 1483 2465 2 247 69 108;9 0 0 0 0 0 0 0 0 0 0 0;13 0 33 33 0 0 0 0 0 9 10 33;1 1 1 1 1 1 1 1 1 1 1 1; -;25689 27 2340 150 1988;9 10 0 0 0;13 14 33 33 33;1 1 1 1 1; -;13 590 240 3 4773 2419 45667 28252 11 3045 24 17 1 4 9;0 0 0 0 31 32 0 0 0 0 0 0 0 1 2;33 34 0 0 39 40 0 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7687 17049 0 30180 0 35045 94 2960 2421 24 1 4 9;0 0 0 23 0 0 0 0 0 0 0 1 2;31 21 0 29 0 11 33 34 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 185706 0 220 2120 8076 2 4730 183 392 4245 0 36079 4245 0 435 4245 16 37 1 87 1 44 0 2 427 262 318 5;0 0 0 0 0 0 0 0 0 0 23 0 0 23 0 0 23 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 0 0 53 0 0 33 34 29 0 33 29 0 33 29 0 33 0 31 0 33 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 8984 503 13054 48 102 390 8984 1525 25562 17 62 594 66 1 399 100 1 961 110 100 0;0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 1 2 0 17 18 0 0;0 11 0 33 0 33 0 11 33 0 0 0 0 0 0 1 2 0 31 32 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;43954 53 53 1539 5 167;21 0 0 1 2 0;27 33 34 1 2 33;1 1 1 1 1 1; -;1367 13 186 254 3 6845 1 4 61;0 0 0 0 0 0 0 1 2;33 0 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1; -;17064 1632 173 2691 13811 394 1125 1896 28 18 116 8 17064 1632 173 2691 13811 394 1125 1896 28 18 105 116 71 0 2 0 1774 884 224 23 13 199 712 729 1256 25;17 18 18 18 18 18 18 18 18 18 0 0 9 10 17 18 18 18 18 18 18 18 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0;13 14 0 0 1 33 33 34 33 34 33 0 13 14 0 0 1 33 33 34 33 34 33 33 34 0 0 0 1 2 2 0 33 34 33 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9383 951 369 55 39629 11614 31610 1 7968 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 33 33 33 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;769 11160 258 103 49882 1 11160 649 502 769 832 5 0 2 0 13172 5469 529 502 55 49741;0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0 0 0;33 0 33 34 33 0 21 22 22 1 2 2 0 0 0 25 26 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;39 85 807 0 1209 611 149 294 1 4 61;0 0 0 0 0 0 0 0 0 1 2;0 0 0 0 33 34 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;30 181 34 484 67 127301 148 496 2031 82 44393 1093 83 1 188 142;0 0 0 0 0 9 10 0 0 0 0 0 0 0 17 0;33 34 33 34 0 13 14 33 33 0 33 33 0 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;39940 19933 2233 1449 59 5305 3227 1397 11217 3765 174 0 2 0 34 1006 5;21 0 0 0 0 0 0 9 0 0 0 0 0 0 1 2 2;27 0 33 34 0 33 34 13 33 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;241 7690 18 305 256 5099 32 5023 17 1 4 9;17 18 18 0 0 0 0 0 0 0 1 2;0 0 33 33 0 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 18093 18350 11777 1971 2417 141085 777248 8282 8 4279 2474 699 1 55579 18093 18350 100 1 110 2334;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 21 22 22 33 33 0 0 29 0 33 33 33 0 11 12 12 33 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;88794 0 26560 0 2 0 74346 40294 0 54969;0 0 0 0 0 0 0 0 0 0;33 0 21 0 0 0 33 0 0 0;1 1 1 1 1 1 1 1 1 1; -;57190 0 2 0 248806 0 26547 0 76084 0 2 0 26111 202 108 0 1182 55 0 548 26 2544 26 2047 2 202 2 35 1288 31 95 566 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 19 20 0 1 2;33 0 0 0 0 0 33 0 33 0 0 0 33 34 34 0 33 0 0 33 0 21 0 35 0 33 0 33 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3039 990 42567 3 321 266 1 4 9;23 24 24 0 0 0 0 1 2;29 30 30 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1; -;62761 0 368 0 313 0 17752 0 237002 0 2 0 8169 0 2 0 140 406 199 1 41 5936;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;35 0 33 0 33 0 55 0 21 0 0 0 33 0 0 0 35 36 0 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3811 3731 45 35190 3 266 0;0 0 0 0 0 0 0;33 34 0 33 0 33 0;1 1 1 1 1 1 1; -;58136 2511 1750 891 2262 103 3 36728 58 7189 4284 24 1 4 209 132;37 0 0 0 0 0 0 37 0 0 0 0 0 17 23 24;45 27 28 33 33 34 0 45 0 33 34 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;61167 3 9153 206 15 413 1 4 9;0 0 0 0 0 0 0 1 2;33 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;10 234 3487 21 50 14021 17 1 10425 12974 5;0 0 0 0 9 10 0 0 1 2 2;33 34 0 0 13 14 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1; -;234 11 0 22215 235 8549 76 0 3 729 3688 62693 998 749 345 0 22 305 0 22 1019 21 29 1443 1590 3 128 0 89 11 998 749 3 345 7 1 4 9;0 0 0 5 6 6 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 33 34 34 34 0 0 33 33 33 33 34 33 0 33 34 0 0 0 0 0 0 33 0 33 0 33 34 33 34 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1907 325 3 6611 3 392 329 0 16554 3 392 329 0 16554 3 661 2 210 2 35 19 2 41 72;29 0 0 11 0 0 0 0 11 0 0 0 0 11 0 0 0 0 0 0 0 0 1 2;33 33 0 15 0 0 0 0 15 0 0 0 0 15 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7195 54 1072 840 3711 0 137 1327 2 783 5463 2 9001 9328 301 2 16353 301 5;0 0 45 46 46 0 0 0 0 0 33 0 0 27 28 0 0 0 0;33 0 53 54 54 0 21 33 0 0 27 0 33 35 36 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16753 721 199 63 2844 27516 24 1 4 9;9 10 10 0 9 9 0 0 1 2;13 14 14 0 13 13 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;139 3732 2648 144 82 8840 813 8136 9812 1313 83 889 1651 462 957 1478 2 24604 5 251 34 4714 5;0 0 0 0 0 0 0 0 11 0 0 0 0 0 0 0 0 1 2 0 9 0 0;0 0 0 0 0 33 34 0 15 0 0 21 22 22 22 33 0 1 2 0 21 22 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7416 77 2156 93 2224 91 590 1 4 9;0 0 0 0 0 0 0 0 1 2;33 0 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;7599 32 7599 1176 10 12 10 514 7599 32 4611 1176 1958 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 0 33 33 33 34 34 0 33 0 33 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7167 190 9372 52 0 138 68 229 1 4 9;17 18 18 18 0 0 0 0 0 1 2;31 32 32 32 0 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;594 8855 97782 4963 7857 23 540 594 25 2 2691 3762 3924 2943 100 2 0 92 191 573;0 17 0 0 0 0 0 0 0 0 9 10 10 0 0 0 0 9 1 2;0 31 33 33 33 0 35 36 0 0 13 14 14 33 33 0 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1531 914 1 322 40 198 334 184 55 1 322 40 334 184 55 198;0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0;33 33 0 1 33 33 33 33 0 0 1 33 33 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;839 848 15794 0 17069 3 987 3804 7154 10 15 30 22 37591 1393 164 3691 106 29 97 3 0 4405 1017 1017 2977 6367 7265 36 22 41 1715 0 2 0 5634 530 0 2 0 956 0 556 0 16044;0 0 0 0 23 0 0 0 0 0 0 0 0 15 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0;0 0 0 0 29 0 33 21 22 0 0 0 0 33 0 0 0 35 36 36 0 0 33 34 0 33 34 34 0 33 34 33 0 0 0 1 2 0 0 0 0 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2069 58 167 10 520 211 24 1 2069 58 167 10 520 211 24 277 9489 490 490 14 988 3567 16 9489 17193;17 18 0 0 0 0 0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 0 33 0 0 33 0 33 0 0 33 0 0 0 0 0 0 33 34 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;338 56443 8835 13 29 97 0 601 7316 30 14856 590 7128 2 255 2 35 19 566 41 72;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 33 33 21 22 22 0 0 0 0 0 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5963 3950 233 2076 73 5485 9120 20843 1437 1158 0 231 390 3816 7048 2671 23 68 25 0 2 0 4662 0 2 1542 3207 5;9 10 10 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 10 0;13 14 14 14 33 34 33 0 0 0 0 0 0 0 33 0 0 0 0 0 0 0 33 0 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13093 1290 41 799 358 4930 21 8 350 3 615 90 614 3 56 24 60 1 59343 15834 1 2874 210 5;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;27 33 33 34 0 33 0 0 33 34 33 0 0 0 0 0 0 0 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;635 319 13109 77 33 3 440 1 1085 115 107 1 136 13109 823 107 77 33 0 1 33 505 31 2 624 463 5 107;9 0 37 0 0 0 0 0 0 0 0 0 0 37 0 0 0 0 0 0 1 2 2 0 0 0 0 0;13 33 45 33 34 34 33 0 33 0 33 0 0 45 0 33 33 34 0 0 1 2 2 0 33 34 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;78111 46 63 4648 7009 1099 4146 160 10148 6566 2813 1568 2030 6564 1 188 268;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24;27 0 0 33 33 0 0 21 22 33 33 0 33 31 0 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;695 94 1511 1248 20 134 12 9868 21 0 2 0 612 295 42;23 24 0 0 0 0 0 0 0 0 0 0 1 2 2;29 30 33 0 0 0 0 33 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;25451 206 17324 1 4 9;0 0 0 0 1 2;0 0 33 0 1 2;1 1 1 1 1 1; -;568 81215 394 42739 0 1496 4809 1181 59506 684 53 53 45 81 5;0 0 0 0 0 9 0 0 0 0 0 0 1 2 2;35 36 33 33 0 13 33 33 0 33 33 34 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;30395 1044045 6657 2 2670 1996 4938 256 110 4073 17;17 0 0 0 0 0 0 0 0 0 0;31 0 33 0 33 33 0 0 33 33 0;1 1 1 1 1 1 1 1 1 1 1; -;1942561 0 1143737 2179 264 1 35 31 95 1 141 5;0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 33 34 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;322 99 529 145 20 156 17 1 4 9;0 0 0 0 0 0 0 0 1 2;1 0 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;22 3 4012 3037 9883 19 7 5764 1 15279 375 375 375 375 375 12225 1 4 271 787 40 0 2 0 1920 4 787;5 6 5 6 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6 6 0 0 0 0 0 17 0;33 34 9 10 33 33 0 21 0 33 0 0 0 0 0 21 0 9 10 10 33 0 0 0 0 31 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;550 38 1874 7490 1080 1075 874 2 2 554 5;0 0 9 0 0 0 0 0 0 1 2;33 34 13 33 33 33 21 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;15 7571 67 2067 4870 3 212 60 2227 77 122 3 4870 3 10 78 8 1101 20 3 184 78 8 3325 73 484 20 3 8 22 10 94 2440 8 346 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 33 0 0 0 0 0 0 0 0 0 0 0 33 0 53 0 0 33 34 0 33 33 34 0 0 0 21 22 22 22 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15892 42 152 385 8382 18849 2469 0 1009 904 3126 7294 13088 2 738 295 36 726 5261 5 23 24170 7 645 25;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0;33 34 0 0 33 0 0 0 33 34 34 0 0 0 33 34 0 1 2 2 0 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 422 302 1769 3 7434 1 4 9;0 0 0 0 0 0 0 1 2;33 34 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1; -;2493 9288 3753 4380 20 5157 1 4 9;23 24 24 0 0 0 0 1 2;29 30 30 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1; -;82 7433 175 56 9533 9339 83 1 188 268;0 0 0 0 0 0 0 0 23 24;0 33 34 34 33 34 0 0 29 30;1 1 1 1 1 1 1 1 1 1; -;1053 687 1 1053 687 629 2817 46 89 11 42 4207 10042 6 89 11 746 93 922 2444 1 4257 5 500 1350;1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0;1 2 0 1 2 33 33 0 33 34 34 33 33 0 33 34 33 0 33 34 0 1 2 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2863 386 24285 2734 10367 4850 1 4177 3100 298 12923 1 33 505 31 2 624 463 5 107 344 372;0 0 0 0 0 0 0 37 38 38 38 0 1 2 2 0 0 0 0 0 0 0;31 0 33 0 21 0 0 45 46 46 46 0 1 2 2 0 33 34 34 33 21 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9298 3 1466 2690 408 1106 595 21666 3 603 8 2812 714 10 91 1 4 9;17 0 17 18 0 0 29 0 0 0 0 0 0 0 0 0 1 2;31 0 31 32 0 33 33 33 0 0 0 33 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;30 8008 164 317 1616 478 580 4206 3 3544 10 20 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 0 33 0 0 33 0 33 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;45202 1 36526 1 5578 58995 1 13153 1 75224 1 61729 1 282999 1 90311 451 1 122583;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 33 33 0 33 0 0 0 21 0 21 0 21 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;112 112 22 413 19268 8 747 10 43827 22 699 90 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 0 33 0 33 0 0 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3678 1635 100 0 2 0 3678 6283 5 0 2 0 3678 1635 5 37339 3 4477 277 1635 4564 0;0 0 0 0 0 0 27 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 33 0 0 0 35 36 0 0 0 0 21 22 0 33 0 33 0 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;600 15332 5516 8331 73895 9347 51 6246 1190 18570 2 3457 10211 5;21 22 0 0 21 0 0 0 0 0 0 0 0 0;27 28 33 33 27 33 0 33 33 0 0 35 36 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1496 16461 381 823 33 17 1 4 9;17 18 18 0 0 0 0 1 2;13 33 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;5720 211 0 2 0 107035 15073 334 382 211;0 0 0 0 0 17 18 0 0 0;33 33 0 0 0 31 32 33 33 33;1 1 1 1 1 1 1 1 1 1; -;28443 3 198418 141912 1758 1591 91 203 1 4 9;0 0 0 0 0 0 0 0 0 1 2;33 0 21 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;1942561 7 645;0 0 0;0 0 33;1 1 1; -;811 2834 28 18 102 1103 5611 436 53 53 2834 64 18704 2653 475 176 10135 182 270 62 127 393 66 3328 1678 0 2 0 34 2437 662 5;17 18 18 18 0 0 0 0 0 0 0 0 31 32 32 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;31 32 32 32 33 33 33 33 33 34 33 0 39 40 40 0 29 33 33 0 21 22 0 33 33 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10012 2 2 1945 235 78 1350;0 0 0 0 0 0 0;33 0 0 1 33 34 33;1 1 1 1 1 1 1; -;10279 6203 2189 323 680 13395 1828 30 88 93 64 24 60 1 4 9;0 0 0 0 9 0 0 0 9 0 0 0 0 0 1 2;21 22 22 22 13 33 33 33 34 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;463 5 570 2151 1013 40640 463 5 177 2151 0 80 3696 2 255 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 33 34 33 33 34 34 33 0 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;598 2216 1870;9 10 10;13 14 14;1 1 1; -;22 236 306 1112 13 124 746 9161 63 4388 56 8 1782 4388 4119 20305 3 288 2161 32 1087 2099 388 302 11 1087 3 138 8 992 93 640 3565 24 7 7 7 22 236 306 1112 13 124 746 9161 63 4388 56;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 0 0 0 0 33 33 0 33 0 0 0 33 33 33 0 33 34 0 33 33 0 0 0 33 0 33 0 0 0 33 34 0 0 0 0 33 34 0 0 0 0 33 33 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;157 27 945 894 1800 161 28 18 2 187 945 894;17 18 18 18 18 18 18 18 0 0 0 0;21 22 22 22 22 22 22 22 0 0 27 28;1 1 1 1 1 1 1 1 1 1 1 1; -;20021 1325 168722 352 259 761 6 20021 1325 168722 6 20021 1325 168722 37 6 20021 1325 168722 260 6 997 259 6 3644 398 2 8676 192 5 0 36234 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0;33 33 33 33 34 33 0 33 33 33 0 33 33 33 33 0 33 33 33 33 0 33 34 0 33 33 0 1 2 2 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2042 366 10 374 4642 1 4 9;0 0 0 0 0 0 1 2;33 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1; -;384 745 49 827 6384 3 526 309 1792 48 367 1029 62 2508 361 66 1 73 1794 496 128;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;35 36 0 33 33 33 34 0 0 0 33 33 0 0 33 0 0 33 34 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;433 703 755 1642 3004 755 15 293 1 4 9;17 18 0 0 0 0 0 0 0 1 2;33 34 0 33 29 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;49 11 13 422 41 30 22 497 865 48 285 1265 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 34 34 55 56 56 0 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;707 199 5801 1 340 2214 569 4462 458 28 18;0 0 0 0 17 18 18 18 18 18 18;21 22 22 0 31 32 32 32 32 32 32;1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 63035 0 8005 2688 711 0 367 711 0 580 2711 130 2 2 2 418 10159 1106 45 8 1825 2136 2109 1 63035 100 1 110 54 115 100;0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0;0 0 0 11 0 21 22 22 0 33 34 0 0 31 33 0 0 0 33 34 33 34 0 0 33 33 0 11 33 0 9 10 10 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2205 335 3 27082 195 2829 6 2205 335 3 27082 137 1327 869 367 6 628 477 2985 1069 3 301 2 6196 300 5;21 22 0 0 0 0 0 21 22 0 0 0 0 0 0 0 21 22 22 22 0 0 0 1 2 2;27 28 0 33 0 33 0 27 28 0 33 21 33 33 33 0 27 28 28 0 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3064 1807832 39724 2574 496 1107 3892 0 2 0 1130 687 0 2 0 34 2113 138 703 0 2 0 956 0 556 0 1157 232;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 0 37 38 33 33 33 0 0 0 21 22 0 0 0 33 34 21 22 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;212 5602 280 236 293 1709 1648 4476 249 462 8 11 234 9 5602 225 11 690 17 37 15 124 17 1 4 9;0 17 18 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 1 2;0 31 32 0 0 33 33 34 33 34 0 33 34 34 13 0 0 0 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2356 412 526 115 107 572 33;0 9 0 0 0 0 0;33 13 33 34 33 33 0;1 1 1 1 1 1 1; -;963 136 7598 156714 3066 4919 24282 6343 1 4 209 132;0 0 37 0 0 0 0 0 0 17 23 24;33 0 33 33 33 33 33 33 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1; -;14 43 164 11 89 11 31 3 8 13 1415 3112 332 8 1380 6104 13 332 216 16 1 73 462 2477 216 1009 2153 55 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 46 46 46 46 0 0 0 0 1 2;0 0 0 33 34 34 33 0 0 33 34 0 0 0 33 33 33 34 0 0 0 53 54 54 54 54 11 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;80 1782 13 643 18691 2005 4939 453 13 3020 1297 1782 54 4105 18691 2005 2144 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 34 21 22 33 33 34 33 0 0 0 0 21 22 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2976 349 2 38703 100 2 38703 42 56 38703 100 2 38703 2642 3622 103 100 0 2 0 956 0 556 0 1157 232;0 0 0 1 2 0 9 0 0 1 2 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0;33 34 0 1 2 0 13 33 34 1 2 0 1 2 33 34 33 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6183 177 0 36 0 5755 6183 1628 957 8 82 5682 666 83 1244 13313 21 59 2 247;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;33 0 0 0 0 33 33 33 34 0 0 21 22 0 33 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 9487 655 16 2380 8875 593 46092 643 2482 14175 39 2999 53 53 3035 1005 2 3516 1171 2 2373 78 100 0 2 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 46 0 1 0 0 0 0 0;0 0 33 0 33 33 0 33 0 0 13 0 33 33 34 33 0 0 53 54 0 1 9 10 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14150 378 13264 3 47;0 0 0 0 0;33 33 33 0 1;1 1 1 1 1; -;10281 194 1013 2978 4967 15 319 17 1 1048 325 382 1 1048 325 5;0 0 37 38 38 0 0 0 0 0 0 0 0 0 1 2;33 0 45 46 46 0 33 0 0 33 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;856 386 2376 190 1280 223 730 1 4 9;0 0 0 0 0 0 0 0 1 2;33 34 33 0 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1; -;358 33 25465 1403 232 0 9719 3788 2846 17595 9272 2846 1787 1084 2 247;0 0 0 0 0 0 21 0 0 0 21 0 0 0 0 17;33 34 34 34 0 0 27 55 0 55 27 0 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;79131 10 22269 155 274 13087 155 1 4 79 9;13 0 0 0 0 0 0 0 0 0 0;17 0 0 0 0 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1; -;2461 46 16493 10654 30 13553 45 8036 214237 1 188 268;0 0 9 0 0 0 0 0 0 0 23 24;33 0 13 33 0 33 0 33 43 0 29 30;1 1 1 1 1 1 1 1 1 1 1 1; -;457 1736 30 20 124 3 544 134 10 109 2199 2974 3 84 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 0 0 0 0 33 0 0 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;351 70 11070 39 106 691 82 37863 7696 12428 172 25245 83 0 2 0 34 191 816 852 1802 334 128 1 45 191 5;0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 17 18 18 18 18 0 0 0 1 2 2;33 34 33 33 34 0 0 13 33 34 33 0 0 0 0 0 31 32 32 32 32 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11963 879 3 19511 2 210257 2896 32 166 2988 1997 8129 4846 3 669 294 0 2 0 364 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;39 40 0 33 0 33 33 0 0 33 34 33 34 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12 2814 1727 165 22 12709 3097 3010 3010 22 30 41130 358 21 91 180 17 1 1139 13035 30 2543 1 5841 523;0 0 0 0 0 0 0 11 12 0 0 0 0 0 0 0 0 0 21 22 0 0 0 0 0;0 0 0 33 34 33 0 15 16 33 34 9 21 22 22 22 0 0 21 22 33 34 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5122 971 32 673 1990 68 2 1323 5122 1990 68 2 1323 1238 2307;0 17 0 0 0 0 0 9 0 0 0 0 9 0 0;21 31 33 34 33 34 0 13 21 22 22 0 13 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 6161 2649 1664 26 24210 3447 225 33 16 1 77 33 3 6161 2649 1664 26 24210 3447 1 2 4121 210 5;0 9 0 0 0 9 10 0 0 0 0 0 0 0 9 0 0 0 9 10 0 0 0 0 0;0 13 33 34 0 13 14 0 0 0 0 33 34 34 13 33 34 0 13 14 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15017 5037 1279 3380 540 3929 11 135099 42 30 3716 45 12133 2 45 81 5;9 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;13 0 0 33 0 33 0 0 0 0 33 34 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11 249 1427 155 254 55486 3810 7119 4497 22514 3041 3 24 17 1 4 79 9;0 0 0 0 0 13 14 14 14 14 14 0 0 0 0 0 0 0;0 0 33 0 0 17 18 18 18 18 18 0 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;24982 54355 10237 5406 4165 46 15252 50953 8 17750 10237 0 2 0 4662 0 2 412 3207 5;17 18 18 18 0 0 0 0 0 0 9 0 0 0 0 0 0 9 1 2;31 0 13 0 0 0 33 33 0 33 13 0 0 0 33 0 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;424 4376 14 14 14 4 94 1144 329 4206 9418 16 16 16 1 4 94 1144 329 43 1 4 47;0 0 0 0 0 5 6 6 6 0 0 0 0 0 0 5 6 6 6 0 0 1 2;33 33 0 0 0 33 34 34 34 33 33 0 0 0 0 33 34 34 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1062 28706 159 420 28 18;17 18 18 18 18 18;13 33 33 34 33 34;1 1 1 1 1 1; -;1528 30 2575 2672 394 0 740 386 1235 1683 1150 7733 82 328 3749 83 218 14719 25245 1 218 131 1 328 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 9 0 0 1 2;0 0 33 33 33 0 33 34 33 34 33 0 0 13 0 0 13 0 0 0 13 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15442 3100 391 7310 107 2275 2882 697 1063920 505 15 124 74 33 12 33 2 33 505 31 2 624 463 5 107;37 38 0 17 18 0 37 38 21 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0;45 46 0 31 32 33 33 34 27 33 0 0 0 33 34 34 0 1 2 2 0 33 34 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 1704 1152 5025 48 14574 195 2469 231 1776 1061 5110 5844 19 4225 1 1704 1152 5025 51 5110 19 1 1945 78 19;45 46 46 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;53 54 54 54 54 21 33 34 0 0 33 33 33 33 0 0 53 54 54 54 54 33 0 1 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6739 2890 555 1964 1288 0 2 0 3010 3010 3010 2 2993 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;53 54 54 54 33 0 0 0 33 34 34 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2178 63323 6515 58 39 6095 0 13333 59 1 2178 3161 1794 2147;1 0 0 0 0 0 0 0 0 0 0 45 46 46;1 33 33 34 0 33 0 33 0 0 1 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2777 77 50 10335 890 121 2989 26 1124 20 156 1 4 9;0 0 3 4 0 0 0 0 0 0 0 0 1 2;33 33 34 53 0 0 33 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;402 109 21 2799 8385 34497 891 2101 171 27134 63200 84 501 4545 722 10 13069 194 27368 8 402 11 20 229 24 17 1 4 79 9;0 0 0 0 0 15 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 33 34 19 20 0 33 33 33 0 0 0 33 0 33 0 0 0 33 21 22 22 22 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7577 1164 5889 1 33387 17140 11904 1 11904 1164 5889 1 7577 1164 5889 4313 1 1102 11904 1164 5889 1 1102 71953 7093 2294 1998 28 18;11 12 12 0 0 0 29 0 29 0 29 0 11 12 12 0 0 9 29 0 29 0 17 18 18 18 18 18 18;15 16 16 0 33 0 0 0 0 0 37 0 15 16 16 0 0 13 0 0 37 0 13 33 0 0 31 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1418 4800 141 534 1 1418 4800 2190 254 3 205 354 2 41 72 972 398 535;21 22 0 0 0 21 22 0 0 0 0 0 0 1 2 0 0 0;27 28 33 33 0 27 28 0 0 0 33 34 0 1 2 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2069 17143 0 1718 390 0 46142 68057 0 13108 51025 44 1 2069 1 110 1930 1 110 54 115;7 8 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;11 12 0 33 0 0 21 0 0 33 0 33 0 33 0 21 22 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 379 360 38 2329 41336 2057 267 1791 1133 381 700 0 2 0 3628 119 98 2 4570 159 67 456 257 2 6249 67 748 435 3125 108;0 0 0 0 0 0 23 0 0 0 0 0 0 0 0 17 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 34 33 33 33 33 33 0 33 33 0 0 0 31 32 32 0 33 34 34 34 33 0 33 34 34 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12 1135 21 385 2 247 204;0 0 0 0 0 1 2;33 34 34 0 0 1 2;1 1 1 1 1 1 1; -;8185 664 552 3 42 134 1229 256 128 60 1 4 9;0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 33 34 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 45620 1525 6186 13793 451 16 45620 32 81 6186 13793 451 1 81 45620 32 81 6186 13793 451 219 33 1 81 45620 32 81 6186 13793 451 572 1 3616 69 5 53 110 131;0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 1 2 2 0 0 0;0 11 33 21 22 22 0 11 0 0 21 22 22 0 0 11 0 0 21 22 22 0 0 0 0 11 0 0 21 22 22 33 0 1 2 2 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2103 10991 438 3 31727 74 2071 403;0 0 0 0 0 0 0 0;33 33 33 0 33 0 33 33;1 1 1 1 1 1 1 1; -;8908 321568 202 2048 46618 53 31 291 53 387 5 8 19 35 31 95;9 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;13 0 33 33 34 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1865 12914 15599 688 1039 3153 3 138 67 571 208 0 2 0 364 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 0 0 33 0 33 34 34 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 431 38 212 1347 64 233 13943 5221 146 7 52 764 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 0 33 34 0 33 33 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9351 5148 13492 12 2444 8 1837 152 41299 8 222 23095 17247 430 59 0 2 0 49 878 1318 85 48 1373 1374 75 0 2 0 3333 14968 100 0 2 0 956 0 556 0 1157 232;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 46 46 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 33 33 34 0 33 0 33 0 0 33 33 0 0 0 0 0 53 54 54 54 54 33 33 0 0 0 0 25 0 33 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2408 324 6649 8 3280 324 119419 6 8156 8 273 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 0 21 22 33 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;277 2 80583 0 90135 0 17790 288 4596 0 6243 17332 617 2 322 5;0 0 0 0 0 0 0 0 0 0 0 0 9 0 1 2;0 0 0 0 33 0 33 0 0 0 0 21 13 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;789 225 3 7743 107 33;9 0 0 37 0 0;13 0 0 33 33 0;1 1 1 1 1 1; -;1815 1016 46 1680 1910 383 120 1602 1 1016 5179;0 0 0 0 0 0 0 0 0 0 0;33 33 0 33 34 33 34 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1; -;1053 2682 6392 3950 701 5538 15 341 0 4963 5967 6392 3950 701 5538 4279 1951 0 36 0 122 557 5 78 282 201;45 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 27 28 0;53 54 0 33 34 0 0 0 0 33 33 0 33 34 0 33 33 0 0 0 1 2 2 35 36 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15 58 1431 4467 17 1431 4467 118 96 1090 1588 20 1 4 79 9;0 0 37 38 0 37 38 0 0 0 0 0 0 0 0 0;33 34 33 34 0 33 34 0 0 33 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;882 46 34 2275 2692 529 52 1041 0 1145 2245 6262 8920 28636 2 247 268;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0;33 0 33 34 33 0 0 0 0 33 33 33 33 33 0 31 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;18493 0 2 0 377380 26 126690 2 7320 51 0 19855 0 430038 0 108183 0 3773 0 63903 0 239 0 18942 0 5517 0 93234 0 519 0 164847 0 1797 0 126690 2 12698 0 3190 0 42962 57135;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 0 0 0 0 0 0 0 33 0 0 33 0 0 0 0 0 33 0 0 0 0 0 21 0 33 0 33 0 21 22 22 0 33 0 0 0 33 0 33 0 33 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6173 148 479 164 13795 43594 55431 10102 58 2 277;9 10 9 10 10 0 0 0 0 0 0;13 14 13 14 14 33 0 33 34 0 0;1 1 1 1 1 1 1 1 1 1 1; -;97 5352 646 931 1036 1036 122546 74 14988 8 2089 32 2285 84;0 0 0 0 0 0 0 0 29 0 0 0 29 0;0 33 34 34 0 0 33 0 33 0 33 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6644 9630 0 70 1081 63 4379 12923 24 1 4 9;0 0 0 0 0 0 37 38 0 0 1 2;21 22 0 0 0 0 45 46 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;1307 39 312 691 82 558 1063 83 479 304 521 3742 1029 1611 361 2 40;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 34 0 0 33 0 0 33 34 34 33 33 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5094 281 4210 207 167 682 1 5094 281 4210 207 167 682 826 2 39 13 211 48 48 48;0 0 0 17 18 0 0 0 0 0 17 18 0 0 0 0 0 0 0 0 0;33 27 28 31 32 0 0 33 27 28 31 32 0 33 0 33 34 33 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;54320 1575 1142 1 49263 0 54320 1575 1142 0 12 1277 0 2407 9184 159 179 5250 514 337 27482 2 189325 0 2 0 111;23 0 0 0 23 0 23 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;33 21 22 0 29 0 33 21 22 0 33 34 0 33 33 33 0 33 34 33 0 0 0 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3091 0 197 0 51 51 0 643 0 51 51 0 54899 148 391 15156 54880;1 0 0 0 0 0 0 0 0 0 0 0 9 10 0 0 0;1 0 33 0 0 0 0 0 0 0 0 0 13 14 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;160 32714 6620 108 1128 1086 30 22469 771 175 199 755 1676 1 4 9;0 9 10 10 0 0 0 9 10 10 10 0 0 0 1 2;0 13 14 14 55 56 0 21 33 34 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1496 641 885 2 0 1496 56481 16791 28 18;9 0 0 0 0 17 18 18 18 18;13 33 34 0 0 13 33 0 33 34;1 1 1 1 1 1 1 1 1 1; -;82 463 5 177 83 10 20 17 0 1 0 297 1852 5 0 1 0 342 686 49 297 1852 491 48 334 128 0;0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 17 18 27 28 28 28 28 0 0 0;0 33 34 34 0 33 34 0 0 0 0 1 2 2 0 0 0 31 32 35 36 36 36 36 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;45445 261 3097 163 7 2599 7 24709 261 0 2 0 2197 77 122 2 545 100 23 12810 7 65 25 0 2;0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0;0 33 0 33 0 21 22 1 2 0 0 0 33 0 0 0 1 2 0 0 33 34 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 515 4476 0 2893 30797 47986 30462 3750 0 988 8465 9793 16 37 6 87 6 44 6 2548 1360 6 802 27 12651 75 281230 559 259 371 2 395 5;0 9 0 0 0 29 0 11 12 0 0 0 0 0 0 0 0 0 0 0 11 12 0 0 0 9 10 0 0 0 0 0 1 2;0 13 33 0 0 37 0 15 16 0 33 33 33 0 33 0 31 0 33 0 15 16 0 13 14 13 14 27 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1379293 0 658177 0 23 4006 25 0 2 0 5523;0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 0 0 0 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1; -;16553 27 726 13499 987 1360 28 18 2 277;17 18 18 18 18 18 18 18 0 0;13 0 27 28 33 34 33 34 0 0;1 1 1 1 1 1 1 1 1 1; -;8536 7979 152 5614 7601 1641 17 18405 16185 3468 1651 31271 2 1353 99;21 0 0 0 0 0 0 0 0 0 0 0 0 1 2;27 33 0 0 0 33 0 33 33 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2010 4758 8 1653039 1527 14901 6679 0 2010 4758 6127 23908 8 550 7 2480 7 2540 0 2 0 81 13 199 570 5;17 18 0 21 0 0 0 0 17 18 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2 2;31 32 0 0 33 33 33 0 31 32 33 33 0 33 0 21 0 33 0 0 0 1 2 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;384 140 32797 914 349 138 508 1 4 61;0 0 0 0 0 0 0 0 1 2;33 34 33 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1; -;16320 421 9528 786 501 223 67 3741 2078 6610 3424 3 1032 194 208 0 2 0 364 5;0 0 37 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 45 33 0 0 0 21 22 22 33 0 21 22 22 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5995 3664 483 46 67 335 3607 5451 4376 21 8 9536 10 1637 56046 2 41 6419 618 2 50 12 312 674 5 23 1781 7 65 25;0 0 0 0 0 21 22 22 0 0 0 0 0 21 22 0 0 0 0 0 1 2 2 2 2 0 0 0 0 0;33 27 28 0 0 27 28 0 33 34 0 33 0 27 28 0 0 33 34 0 1 2 2 2 2 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1066 0 208839 0;21 22 22 0;33 0 0 0;1 1 1 1; -;4085 881 189 8 2308 284 12 6117;0 0 0 0 0 0 0 0;33 34 34 0 33 33 34 0;1 1 1 1 1 1 1 1; -;48649 1744 11 64 525 413 3 2915 442 8 10 12 10 134 10 5401 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 33 34 0 0 0 33 0 0 33 34 34 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;789 2737 178 138 1 2087 5;17 18 18 18 0 1 2;13 33 34 34 0 1 2;1 1 1 1 1 1 1; -;222 1724 0 22 366 10 12 3157 136 90 3 1810 59 59 59 59 1 13392 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 21 22 22 33 34 33 34 0 33 0 0 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;90296 2998 1604 46 441351 8 13764 14795 45 3 10505 54 1026 1368 2318 2 31 95 2 2697 5;0 0 0 0 9 0 9 10 0 0 0 0 0 17 0 0 0 0 0 1 2;21 22 22 0 13 0 13 14 33 34 33 34 34 33 33 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6778 357 798 6878 16911 23 1488 25 1 8453 1 280 1 8926 31 1 1542 280 5 1 1542 77 1606 3 280 255 411;0 0 0 0 0 0 0 0 0 9 0 0 0 1 2 0 1 2 2 0 9 0 0 0 0 0 0;33 33 34 33 33 0 0 0 0 33 0 33 0 1 2 0 1 2 2 0 13 0 33 0 21 22 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;51982 652 11 16892 1657 11546 7383 10467 134 12 371 15 143 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;33 0 0 0 33 0 0 0 0 33 34 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6568 9566 4786 8951 10 20 156 1 4 9;0 0 0 0 0 0 0 0 1 2;33 0 0 33 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1; -;106429 148 84125 1341 14210 1299 82 13 1235 106 4392 312 4308 83 349 3 0 1363 508;9 10 0 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 14 33 13 14 33 0 0 0 0 0 0 33 0 33 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 2 0 18468 0 110918 0 4043 390 0 110918 100271 0 85 7 52 2083 0 2644 0 2 0 9664 10384 444 299 101 4588 0 2 0 9664 1412 5;0 0 0 0 0 19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 10 7 8 0 0 0 0 9 1 2;0 0 0 0 0 25 0 33 0 33 0 0 33 33 0 0 0 0 0 0 33 0 0 0 21 22 22 33 34 33 0 0 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9075 1003 13956 220 2 129 4400 287942 1403 289 1702 3519 282 655 1 9075 1003 13956 220 2 129 4400 287942 1403 289 1702 3519 282 655 2 9075 1003 1 3112 3065 5;45 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 46 0 0 0 0;53 54 33 0 0 0 0 0 21 22 0 33 33 33 0 53 54 33 0 0 0 0 0 21 22 0 33 33 33 0 53 54 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5795 1957 2224 10 15 441 203 17 1 11 217 846 1021 1 595 217 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 0 0 33 0 0 0 1 2 2 2 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6313 10 378 2370 6 222 4883 32 8468 1 4 130 112;0 0 0 0 0 0 0 0 0 0 1 2 2;33 0 33 33 0 0 33 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;797 1193 324 0 4816 0 45 16961 7874 3 3708 10 250 3 1 4 9;0 0 0 0 45 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 0 33 0 0 41 42 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;24818 3221 938 2090 36071 812 16212 3392 85 2630 2723 0 21244 618 414 1279 306 1220 30 726 4123 259 1 34 297 5 6518 6518 272 297 411;17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0;31 27 28 33 33 33 33 0 0 0 33 0 33 34 33 0 0 33 0 0 33 34 0 1 2 2 0 0 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;37829 93 3486 20 1033 10 20 1 4 9;0 0 0 0 0 0 0 0 1 2;33 0 33 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1; -;10659 4408 52 10 12 10 1581 1 4 9;0 0 0 0 0 0 0 0 1 2;21 22 22 33 34 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1; -;330 3994 174 2437 28 18 1 334 128 277;17 18 18 18 18 18 0 0 0 0;13 0 0 33 33 34 0 33 34 0;1 1 1 1 1 1 1 1 1 1; -;321 13662 0 1690 0 7914 45 3 1564 246 0 1795 26 3587 0 416 0 2 0 364 5;0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 9 0 0 0 33 33 34 33 34 0 21 22 22 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;126 6842 169 107 109 1761 96 17683 20383 1 4 79 9;0 17 18 18 0 37 0 0 0 0 0 0 0;0 31 32 32 0 33 0 33 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1354 113 169 287;17 18 18 18;31 32 32 32;1 1 1 1; -;215 2702 929 38 3496 30 151 764 2376 3 84 1680 96 45 804 3 84 22 236 2921 11413 84 234 70 403 162 2702 249 22 203 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 33 34 0 33 0 33 0 0 33 0 0 0 0 0 33 34 33 33 0 0 0 33 0 33 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;32633 46143 360 1 316 197;0 0 0 0 17 18;27 31 33 0 1 2;1 1 1 1 1 1; -;14 4241 1630 5432 16;0 23 24 0 0;0 29 30 33 0;1 1 1 1 1; -;564 4836 38599 148 13 1597 1594 4659 4521 20626 12518 1773 10868 117 848 0 14918 540 152 1918 2 255 2 35 19 2 41 72;0 9 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 14 13 14 0 33 0 33 33 33 33 33 33 0 0 0 0 0 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1209 967 1336 138 343 5117 23503 2225 1 4 61;0 0 0 0 0 0 0 0 0 1 2;33 0 33 34 33 33 33 21 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;173561 0 5508 22 10 37788 2 315 2 35 19 2 41 72;21 0 0 0 0 0 0 0 0 0 0 0 1 2;27 0 0 33 34 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;33784 31608 4882 5383 51 65007 0 1321869 23 4430 8330 4356 4532 9769 5186 10292 14497 5832 0 2 0 3076 638 300 5;21 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;27 28 33 33 0 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2363 20120 950 16 1 2363 20120 950 37 1 2363 20120 950 104 1 2363 20120 950 87 1 395 5 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;0 33 21 22 0 0 33 21 22 33 0 33 21 22 33 0 33 21 22 31 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 154 68 16 971 39464 787885 2 933 299 971 642 107168 4136 62 39464 66 94 2 631 5;0 0 0 0 17 23 0 0 9 0 23 24 24 24 0 23 0 0 0 1 2;0 0 0 0 31 33 0 0 13 33 29 30 30 30 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13374 0 36 0 3762 823 107 19039 3290 33 7 4483;1 0 0 0 9 0 0 0 37 0 0 0;1 0 0 0 13 0 33 33 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1; -;9309 1539 383 18 10 115 250 3 18 17 0 2 0 5350 1071 3 1487 0 2 0 605;17 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1;31 32 33 34 0 0 0 0 33 0 0 0 0 33 33 0 33 0 0 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10700 96229 408 2 157 732 894 192 119 28 18 2 34 997 516 5;0 0 0 0 17 18 18 18 18 18 18 0 1 2 2 2;33 0 33 34 13 0 0 33 34 33 34 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;17737 21075 207450 30 24269 56;0 0 0 0 0 0;11 21 21 0 0 0;1 1 1 1 1 1; -;8322 9806 713 14874 1599 20 19110 0 14874 19110 15 800 1 4 9;45 46 46 0 0 0 0 0 0 0 0 0 0 1 2;53 54 54 21 33 0 33 0 21 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;18017 21033 12183 126 96 900 1705 201506 1 4 9;0 0 0 0 0 0 0 0 0 1 2;33 34 33 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;65987 6888 1713 3219 4223 7808 6346 125 2 8378 2631 5;9 0 0 0 0 0 0 0 0 1 2 2;13 33 33 33 21 22 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1; -;14 87 104 0 64 539 1165 0 179 586 45610 8006 5858 711 62 672 5813 66 22891 76 16 37 1 87 1 44 0 2 427 262 318 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 31 33 0 0 0 33 0 0 33 31 33 34 0 0 33 33 0 21 22 0 33 0 31 0 33 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 2 0 7942 401 565 3520 425 1459 38 183 0 2 0 543 27 754 2169 183 2073 0 2 0 543 1412 5;0 0 0 0 0 9 10 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 0 0 0 0 13 14 14 33 0 33 0 0 0 0 0 13 14 0 33 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 3 94 1434 7722 886 3495 8 2721 13 303 193 135 362 687 12 1927 21 8 15 441 2881 0 2 0 111 1722 922 0;0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0;33 34 29 30 33 0 33 0 0 0 33 33 0 33 33 33 34 0 0 0 33 0 0 0 0 31 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1373 232 14099 203 232 6908 356 152 827 298 1234 2824 1499 64 26870 2482 232 70 1550 1894 24 60 1 4297 146 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 7 8 0 0 1 2;33 0 33 33 34 33 0 0 33 0 33 0 33 0 33 0 0 0 0 0 0 0 0 11 12 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1169 4878 135 992 1001 241 527 8 15 1743 17 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 33 34 33 0 0 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1262 9 350 699 22 999 22 10 11 365 462 3 8 999 348 12 8192 350 171 2444 21 166 350 2966 11 145 16580 21 8 22 542 15 143 432 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 35 36 0 33 34 0 33 34 0 0 0 0 0 33 0 21 22 22 0 0 33 33 34 33 0 0 33 34 34 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7236 44370 5360 486 0 2 0 558 352 259;0 0 23 24 0 0 0 0 0 0;33 0 29 30 0 0 0 33 33 34;1 1 1 1 1 1 1 1 1 1; -;92939 8881 251188 2 1978 460 4152 159 779 176 3884 1343 0 2 0 364 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 0 33 13 14 33 34 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;181 350 12194 5587 3 529 806 4074 201 0 2 0 4118 253 0;0 0 0 0 0 0 0 0 0 0 0 0 5 6 0;0 0 0 33 0 33 34 33 34 0 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 7592 54 9166 6 42 421 54 150956 348 83 6188 23 25 84 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 0 0 33 34 0 33 0 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;81 6020 85 1219 171 6 6020 85 67 1678517 52 454 6 705 6 3249 0 6020 1 2956 5;0 23 24 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 1 2;0 29 30 33 34 0 29 30 0 33 0 33 0 33 0 29 30 30 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14828 9717 16632 13111 5187 377 0 53225 176 1277 14644 2 1533 31 3838 100;17 0 0 0 0 0 0 0 0 0 23 0 1 2 0 0;31 33 0 25 26 0 0 33 0 33 29 0 1 2 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 2 0 299 94 0 2 0 6652 63973 299 94 0 2 0 6652 1412 5;0 0 0 0 0 23 24 0 0 0 9 0 23 24 0 0 0 1 2 2;0 0 0 0 0 29 30 0 0 0 13 13 29 30 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6452 46063 559 3915 0 1163 13583 1279 269 170186 38968 22669 36 559 3915 36 1163 13583 36 6452 1 114 268 1 114 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 1 2;33 33 33 33 0 21 22 0 33 0 33 33 0 33 33 0 21 22 0 33 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8744 824 1313 1680 50 24;0 0 0 0 0 0;33 15 16 33 0 0;1 1 1 1 1 1; -;13180 23421 0 4757 23421 0 575 2798 1737 0 845 1737 0 8461 206 1 88 12975 12359 1 88 761 5;9 10 0 9 10 0 0 0 0 0 0 0 0 31 32 0 0 0 0 0 1 2 2;13 14 0 13 14 0 33 33 34 0 33 34 0 39 40 0 13 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;402 80 309 259141 232 92297 1457 289 0 34376 64526 0 490 1257 7 7 7 7;0 0 0 0 0 45 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 33 34 53 33 0 0 53 54 0 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 14167 62 83603 66 74184 2 50 1511 87670 362 7159 851 2824 138 969 518 669 86 16 37 6 87 6 44 6 669 86 6 88 11466 12680 1998 28 18 2 395 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 0 1 2;0 33 0 21 0 0 0 33 34 33 33 33 33 0 33 33 0 29 30 0 33 0 31 0 33 0 29 30 0 31 32 32 32 32 32 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 933 13162 1046 16 2 3207 933 299 191;0 9 0 0 0 0 0 9 0 0;0 13 13 14 0 0 21 13 33 34;1 1 1 1 1 1 1 1 1 1; -;4690 286 3636 837 1172 5402 3 4998 32 1972 0 36 0 4741 27 169 287;0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18;33 33 33 34 33 34 0 33 0 33 0 0 0 31 32 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;36279 67 7781 938 7941 32 8110 4760 0 2 0 421 499 480 480;11 0 0 0 0 0 0 0 0 0 0 1 2 2 2;15 0 33 34 33 0 33 33 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;88 467 344 800 99 614 344 800 4657 99 13 124 2681 1413 8 373 341 160 1182 143 442 17 0 2 0 612 295 42;17 18 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;31 32 21 0 0 0 21 1 2 0 0 0 33 33 0 0 0 0 33 0 33 34 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1325 764 3 4289 3579 15 1335 1 4 9;0 0 0 0 0 0 0 0 1 2;33 0 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;17285 21544 18 12775 249 5970 18 1 4 9;0 0 0 0 0 0 0 0 1 2;33 34 33 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1; -;151316 445 437 0 455 1135 34194 0 151316 1459 1068 2 325 2 35 19 2 41 72;0 39 40 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 47 48 0 33 34 34 0 33 33 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2047 2 40733 2 467 463 5 46 15066 67 9753 2 26514 4966 0 7 679 1 1115 40;0 0 21 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 5 0;35 0 27 0 33 34 34 0 33 0 33 0 31 33 0 0 33 0 9 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;773 33 8 22 236 993 129 353 12768 8 1160 237 2681 64 1495 5005 8 22 1200 227 21 7125 117 353 122 3 101 8 1160 215 212 5005 237 165 12 371 8 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 0 0 0 0 0 1 2;21 22 0 33 34 33 0 0 33 0 0 33 33 0 33 33 0 0 0 0 0 0 0 33 34 0 0 0 0 0 0 31 32 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1354 67 2423 401 10613 1148 1602 0 2 0 69 633 0 2 0 7071 27 401 153;9 0 17 18 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18;11 0 31 32 33 21 22 0 0 0 33 33 0 0 0 31 32 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;41 7233 3 9239 0 8099 905 0 155 5545 3 1804 25214 21 0 78 2 78 2 35 19 2 41 72;45 46 46 46 0 0 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 0 1 2;25 26 26 26 0 33 34 0 0 33 0 33 45 0 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;250 30 4 47 164 369 6865 17 1 4 9;0 0 1 2 0 0 0 0 0 1 2;0 0 1 2 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;13 644 17072 2226 8 1459 12 2221 2554 8 48120 341 6185 59 0 2 0 649 848 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 34 0 33 0 33 0 0 0 0 33 0 33 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;19 6253 0 14 3437 12 2841 859 16 197420 0 9469 0 23850 0 17822 2670 6067 55 7210 574501 3916 5587 1 19 1 1945 78 19;0 0 0 0 9 0 0 0 0 0 0 19 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 0 13 33 34 34 0 0 0 33 34 34 0 27 28 33 0 0 0 33 33 0 33 0 1 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5251 244 3669 42 201879 135 181 978 7359 15253 15158 1021 2808 217 2 2 34 1590 5 0 1038 169 1590 3688 5145 0 835 77 50 300 2853 235 411;0 0 0 0 21 0 0 0 0 9 0 0 0 0 0 0 1 2 2 0 17 18 18 18 0 0 0 0 0 0 0 0 0;33 34 34 34 27 33 34 21 22 13 33 33 34 34 0 0 1 2 2 0 31 32 32 32 33 0 33 33 34 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;79576 3 6864 1108 873 225 11 2329 17 2130 1146 17 1 4 9 0;0 0 9 10 10 0 0 0 0 0 0 0 0 1 2 0;33 0 21 22 22 0 0 33 0 33 33 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 464 158537 15860 16 59351 1942561 996556 251 0 46229 376355 124019 1 15860 251 1946 1 836 1 1481 1 743 734 743 734;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;0 0 27 33 0 25 0 0 0 0 33 0 33 0 33 0 21 0 33 0 1 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1234 5478 44 1 1234 5478 201 26 3068 68 0 2 0 318 5 3920 35 1197 44;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0;33 33 33 0 33 33 33 0 33 0 0 0 0 1 2 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;26640 190 972 1404 3515 3 10075 2 1404 2 10685 5 36 56788 7 65 7 645;0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0;33 35 36 33 34 0 33 0 33 0 1 2 0 0 33 34 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14164 737 1919 23 373490 0 9497 737 1919 2162 25 1690 228 7 117 1473 0 695 55 193 40 0 2 0 1055 1342 40 0 2 0 1055 193 40 199;31 23 24 0 23 24 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 29 30 0 0 0 0 0 0 0 0;39 29 30 0 29 30 30 29 30 33 0 0 21 22 22 33 0 33 0 21 22 0 0 0 37 38 33 0 0 0 33 34 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3536 249 79 332 29 11 187 369 601 547 220 11 604 3 23554 8 11 20 33 3 606 1308 17 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 21 0 0 0 0 0 0 33 0 0 0 0 33 0 33 34 33 34 33 33 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;279 39 13 82 20174 83 0 350 10 77 58 1469 3 1109 365 0 2 0 463 5 0 2 0 16257 2024 5;17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;31 33 34 0 33 0 0 0 0 0 0 33 0 33 34 0 0 0 33 34 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;237 651 5175 82 4401 83 497 6269 26931 46 7618 3496 7232 1517 337 82 937 180 136 2477 83 0 2 247 268;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0;33 33 34 0 0 0 21 22 0 0 33 34 33 33 34 0 33 34 34 0 0 0 0 31 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2214 24527 19572 44 8 340 2214 24527 19572 7216 865 517 639 68 8 1884 684 68 8 4329 68 8 10849 68 8 7131 68 0 2 0 340 3443 499;9 10 10 0 0 9 9 10 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 1 2;0 0 33 33 0 13 0 0 33 13 14 33 34 34 0 33 34 0 0 33 0 0 33 34 0 33 0 0 0 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 102 38 471 97 1876 181 4324 637 269 2706 3427 2329 637 19869 19256 58 16 2 3411 116;0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 34 34 34 34 13 33 34 33 33 33 33 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1373 16 4010 22 7703 74699 1661 331 10 1961 1 9691 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 29 0 0 1 2;0 33 0 33 0 0 33 0 33 34 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;461 17571 314 28 18 15 124 17 23 137 116 74 719 1238 1183 8 1195 648 74 544 578 74 1815 2314 25 0 2 0 1774 884 224 23 13 199 712 729 1256 25;17 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0;13 27 33 33 34 0 0 0 0 21 33 0 33 33 34 0 33 33 0 33 33 0 33 33 0 0 0 0 1 2 2 0 33 34 33 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2175 21520 501 160 3628 107 15 124 74 33 12 33 17;37 38 38 0 17 18 0 0 0 0 0 0 0;45 46 46 0 31 32 0 0 0 33 34 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;688819 0 11 91 688819 6 688819 11870 17932 2 1513 5 0 11392;21 0 0 0 21 0 21 0 0 0 1 2 0 0;27 0 0 0 27 0 27 33 34 0 1 2 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;174 936 457 118 1369 15 1375 0 19307 50 1369 3 10463 30 3278 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 0 33 0 0 33 34 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6899 36393 86 1078 6 6899 36393 86 1078 0 6 6899 36393 86 1078 0 2 693 51 26 26 163 7 12830 18842 12260 7 65;23 24 24 0 0 23 24 24 0 0 0 23 24 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0;29 30 30 33 0 29 30 30 33 0 0 29 30 30 33 0 0 25 26 26 26 33 0 33 33 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 22 3 446 16 1 2986 101 11 3069 1 2986 110;0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 34 33 0 0 21 0 21 22 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1; -;133 34 13498 133 287 411 128;0 9 9 0 0 0 0;0 13 13 0 33 33 34;1 1 1 1 1 1 1; -;102 38 82 13 293 13 175 83 4951 312 2547 4610 0 10148 1148 11 6101 2094 1 172 3713 1 588 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 33 34 34 34 0 33 33 34 33 0 21 22 33 34 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;309 699 3 42 606 2233 413 30 59038 12749 30 10262 164 5883 317 3209 236 22 1448 24 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 35 36 0 33 33 0 0 33 33 0 33 0 0 0 33 21 22 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;198 38 10053 0 14500 40 890 3092 1305 0 999 16419 1137 4193 764 0 2 0 6521 69 131;0 0 5 6 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 9 10 10 33 0 0 0 0 0 33 0 33 0 0 0 0 21 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;46684 3 3572 1710 7500 4422 21 7904 2776 15 1894 1 4 9;0 0 23 24 24 0 0 0 0 0 0 0 1 2;51 0 29 30 30 0 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 718 2558 231 1540 48 550 2 360 186 38 511 304 6929 626 56 62 1614 723 55 66 39 127 1276 8285 323 3 3369 1 3185 5;27 28 28 28 28 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 34 34 0 33 0 33 33 34 33 34 33 34 0 0 33 33 0 0 21 22 22 33 34 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;139 3463 144 3463 46 497 868 28557 2085 186 7934 62 127 66 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 0 33 0 33 34 33 34 34 33 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;155 3366 9810 32 3366 3042 913 42410 21175 12 241 31987 1 4 9;0 0 0 0 13 14 14 14 0 0 0 0 0 1 2;0 0 0 0 17 18 18 18 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 35333 48 18816 3 156 1 4 130 112;0 0 0 21 0 0 0 1 2 2;35 36 36 27 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1; -;26760 53718 0 2 0 65773 0 428396 0 35047 0 537 0 7228 7 0 25804;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;31 0 0 0 0 33 0 0 0 21 0 0 0 33 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 943 45 1703 14 17256 2785 16 3 7293 266 1 943 2197 43 1 4 47;0 0 0 0 0 11 12 0 0 0 0 0 0 0 0 0 1 2;0 25 0 33 0 15 16 0 0 33 33 0 25 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;778557 1 538277 24505 1 122623 7 679 1 1115 40;0 0 0 0 0 0 0 0 0 0 0;0 0 0 33 0 33 0 33 0 9 33;1 1 1 1 1 1 1 1 1 1 1; -;32691 20555 4921 987 2746 486 4806 67 8041 1 221311 10013 1 1655 3115 3168 5;31 32 0 0 0 0 0 0 0 0 21 0 0 1 2 2 2;39 40 33 34 33 34 33 0 33 0 27 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;110 1944 14815 74 799 14815 993 8 2812 1277 8 7963 33867 59 1 3539 1527 832 3 101 1936 337 1 1552 868 1936 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 22 0 0 0 0 0 0 9 1 2 2;33 0 0 0 0 0 33 0 33 33 0 33 33 0 0 0 33 33 0 33 34 33 0 13 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;545 29215 75 9673 1330 216 6939 1095 0 471 42 152 19712 1 188 69 108;9 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 33 34 33 33 0 0 33 0 33 0 0 33 0 9 10 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;798 165 317 1788 5751 688 659 2884 8 365 3 3570 96 3740 688 8 365 3 10 12 10 166 190 3 89 1351 156 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 33 33 0 0 33 0 0 0 33 0 33 0 0 0 0 33 34 34 0 0 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8766 27 275 2008 139030 976 359 36 8766 27 275 2008 139030 976 359 488 1475 1102 23316;0 0 0 0 31 0 0 0 0 0 0 0 31 0 0 0 0 0 0;13 0 0 0 39 33 34 0 13 0 0 0 39 33 34 0 0 13 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2673 78227 922;0 0 0;33 33 0;1 1 1; -;7674 15331 3 547 1092 20 203 8 710 2921 6678 6988 8 975 64 20 547 3 7674 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 0 33 33 0 0 0 0 33 33 33 0 33 0 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;20 10 28227 26960 26 26960 2080 883 1 20 10 28227 26960 26 26960 2080 883 334 128;0 0 9 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0;0 0 13 33 0 33 33 33 0 0 0 13 33 0 33 33 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22267 3 828 0 23 1808 25;19 20 20 0 0 1 0;35 36 36 0 0 1 0;1 1 1 1 1 1 1; -;750 168 1827 1029 1348 10 162 99 206 17 1 4 9;39 40 0 0 0 0 0 0 0 0 0 1 2;47 48 33 33 33 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;5658 2882 107 2 277;17 18 18 0 0;31 32 32 0 0;1 1 1 1 1; -;1751 55680 27496 17695 6717 0 2 0 1261 1634 491 547 491 0 2 0 192 55 0 2 0 235 55;0 0 0 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 33 13 14 0 0 0 35 36 36 33 34 0 0 0 33 34 0 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;96832 40593 361 2 39 57 535;0 0 0 0 0 0 0;0 0 33 0 0 0 0;1 1 1 1 1 1 1; -;16067 2 21820 740 4136 0 49347 1169 5852 252 8311 1440 86 0 2 0 13858 2 21390 334 128;0 0 0 0 0 0 0 0 0 0 23 24 24 0 0 0 0 0 0 0 0;33 0 33 33 0 0 0 33 33 34 29 30 30 0 0 0 31 32 32 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5351 1717 98 102 38 1186 561 296 176 2487 2779 1 5351 1717 98 1186 5;17 18 18 0 0 0 0 0 0 0 0 0 17 18 18 1 2;31 32 32 33 34 33 34 33 0 33 33 0 31 32 32 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 293 1982 136 158 54 39777 2 2 12 333 7658 3 23445 4090 2 247 280;0 0 0 0 0 0 9 0 0 0 0 0 0 11 0 0 0 0;0 0 33 33 34 0 13 0 0 33 34 34 0 15 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;92 362 94 183 3 1169 22749 23169 15417 2 662 150 0;9 0 0 0 0 0 0 0 0 0 0 0 0;13 33 33 34 0 29 30 33 33 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;5201 13 3675 23 146 1049 11056 5201 13 957 25 6 5201 280 282 0 2 0 3184 2782;17 18 18 0 0 0 0 9 0 0 0 0 9 0 0 0 0 0 0 0;13 0 33 0 0 33 33 13 33 34 0 0 13 33 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;408 0 28585 0 368 0 4855 2 133378 0 329310 6 0 408 0 28585 0 368 0 4855 1889 40 6 579 40 0 2 0 2326 202;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6;0 0 33 0 33 0 33 0 27 28 28 0 0 0 0 33 0 33 0 33 9 33 0 33 33 0 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;212094 0 34386 0 12050 6277 0 8361 3624 5;21 22 22 0 0 0 0 1 2 2;27 28 28 0 25 26 0 1 2 2;1 1 1 1 1 1 1 1 1 1; -;22 3 4027 223 56 413 3 130 9704 121 4027 130 8 402 4027 130 10 1123 93 8185 1233 2547 130 432 17 10 6385 3 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 22 0 0 0 0 33 33 0 33 34 0 33 33 34 0 0 0 33 0 33 33 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;114 11365 34 850 26691 393 1631 1 35 4242 1 114 5;17 0 9 0 21 0 0 0 0 0 0 1 2;31 0 13 0 27 0 0 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;176503 3965 71 1865 5191 7824 6955 1126 1856 55 0 176503 3965 71 1865 5191 7824 6955 1126 1856 55 2 37294 1309;0 17 18 0 0 0 0 0 0 0 0 0 17 18 0 0 0 0 0 0 0 0 0 0;21 31 32 33 33 33 34 0 33 0 0 21 31 32 33 33 33 34 0 33 0 0 27 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1597 6591 3934 866 16 1 1597 6591 3934 866 44 1 37 0 2 0 322 5;0 0 0 23 24 0 0 0 0 23 24 0 0 0 0 0 0 1 2;0 33 33 29 30 0 0 33 33 29 30 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 11 964 746 8 236 27680 930 29 97 8 225 11 17569 199 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 33 0 0 33 33 34 0 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34 252 4775 415 46 2149 808 415 601 1594 3460 804 51545 14153 0 36 4506 415 36 14153 36 3460 804 1 114 268 1 114 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 13 14 0 0 0 0 0 0 1 2 0 1 2;33 34 0 0 0 33 21 22 0 0 33 0 0 33 0 0 17 18 0 33 0 33 0 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 4249 312 99 46 17612 16 48981 2257 14574 985 264 2 78 2 35 19 2 41 72;0 45 46 46 46 46 0 19 0 0 0 0 0 0 0 0 0 0 1 2;0 53 54 54 0 33 0 25 33 21 33 34 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4039 678716 68339 26 140607 0 736353 0 169765 8191 782 34581 46797 34286 1 1980 19 1 1236 1 170 832 71 36 34 3355 259 5 20533 22799 7 645;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 21 0 11 0 0 0 33 0 0 0 33 33 0 33 33 0 33 0 33 33 33 0 13 21 22 0 33 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34 591 3 1633 1 187 34 591 3 1633 1 2571 34 591 3 33 1334 1 1633 2919;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 0 33 0 0 21 22 0 33 0 33 21 22 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9976 1634 491 3931 2454 750 1163 1344 1283 51 129 173 129 12795 85 421 9168 324 491 2321 23 1488 25 1 188 69;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 33 33 33 0 0 33 0 0 0 0 33 0 0 0 33 34 33 0 0 0 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1844 238 453 1834 453 5342 3 42720 654 3 654 2821 1903 121 9457 265 228 7 2480 5802 62 151 842 177 26224 8708 25 8 377 52 265 228 7 2835 5802 62 151 842 177 22210 8708 25 8 252 45 8 9457 74 377 52 3 0 2 0 3022 43 391 361 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;0 0 0 33 0 33 0 21 22 0 21 22 33 0 33 0 0 0 21 55 0 33 0 0 0 0 0 0 33 34 0 0 0 33 55 0 33 0 0 0 0 0 0 0 0 0 33 0 33 34 0 0 0 0 1 2 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2009 3 4925 17 650 225 11 34453 3 17 2312 225 11 34453 3 17 1 4 9;29 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 13 0 0 33 0 0 33 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2174 2174 108 1384 1990 0 2 0 2174 757 1912 5;9 9 10 0 0 0 0 0 9 1 2 2;13 13 14 33 33 0 0 0 13 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1; -;83940 16588 28344 1832 90 0 12950 5833 791 11023 0 60853 2 167 2 35 19 2 41 72;0 0 0 0 0 0 21 21 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 33 34 0 27 27 21 22 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6651 76 611 4346 7 2210;23 24 0 0 0 0;29 30 33 34 21 22;1 1 1 1 1 1; -;1198 117645 46 4659 55638 7515 10175 0 1948 4717 7294 36661 112504 1 69 108 1 12925 5;0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 27 0 33 33 0 33 0 0 0 0 33 33 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4813 56745 3016 1898 76 1 56745 3016 1898 76 1 293 4813 56745 3016 1898 76 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;0 33 21 22 22 0 33 21 22 22 0 0 0 33 21 22 22 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;35718 211 4731 891 200 29 97 118 455 28006 8488 60 891 118 106 29 97 30 4541 221 795 2554 766 17930 8 922 11508 891 6207 795 19191 8 1621 2895 1 4 9;37 38 38 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;45 46 46 33 0 0 27 28 0 33 0 0 33 0 35 36 36 0 33 0 33 0 0 0 0 0 33 33 33 33 33 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3269 41863 0 222 162 29 3137 440 0 2 0 8293 5;23 24 0 0 0 0 0 0 0 0 0 1 2;29 30 0 0 0 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2372 158 16 1 2372 158 37 1 2372 158 104 1 2372 158 87 1 395 5 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;0 13 14 0 0 13 14 33 0 13 14 33 0 13 14 31 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;50 27266 54 3161 1789 0 53266 146 2 220 224 1289 403 62 5897 46 27187 0 546662 0 67574 2 167 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 21 0 21 0 0 0 0 0 0 1 2;25 26 26 26 26 0 0 0 0 0 0 33 33 0 33 0 27 0 27 0 27 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5107 3500 5491 1784 691 283 393 4817 1190 42597 46 4423 1720 82 17556 2418 83 5107 3500 4817 1190 2 34 0 30638;0 0 0 0 0 0 0 0 0 0 0 17 18 0 0 0 0 0 0 0 0 0 9 0 9;33 33 33 33 0 0 0 33 33 33 0 31 32 0 0 33 0 33 33 33 33 0 13 0 13;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 812 1716 3672 10833 1126 0 630 1716 3672 4595 535 1126 16 37 6 87 6 44 6 10833 1126 6 598 27 8324 10453 28 18 2 395 5;0 0 0 23 24 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 31 32 0 17 18 18 18 18 18 0 1 2;0 33 0 29 30 30 0 33 0 33 21 22 22 0 33 0 31 0 33 0 39 40 0 31 32 32 32 32 32 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;19908 0 257687 6 0 62749 0 282554 0 21049 0 44167 0 36923;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 21 0 0 33 0 0 0 33 0 33 0 53;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8197 5199 1065 445 1715 282 1 8197 5199 1065 445 147 1 8197 5199 1065 445 1715 440 0 2 0 4228 5;17 18 9 10 0 0 0 17 18 9 10 0 0 17 18 9 10 0 0 0 0 0 1 2;31 32 33 34 33 33 0 31 32 33 34 33 0 31 32 33 34 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;80985 3706 483 36 273 6 80985 2139 0 2 0 744 1538 0 2 0 744 960;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 31 0 0 33 0 33 33 0 0 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;971 94 2505 118 1511 56 11 2973 73 226 3098 2 34 186 5 2 34 1109 339 411 128;23 24 0 0 0 0 0 0 0 0 0 0 1 2 2 0 9 0 0 0 0;29 30 33 0 33 0 0 33 0 33 34 0 1 2 2 0 13 33 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;128 314 5444 252 128 838 0 1387 55 0 622 12214 2 1055 2915 5474 311;0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0;33 34 33 0 21 22 0 29 0 0 0 0 0 33 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;995 2892 995 49 44854 5776 1871 21 19932 73 148 48 1 10430 60724 629 23035 2742 1 10430 19 1 10430 50 534 1 6719 201 1 3016 97 1 19984 7 65;23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;29 30 0 0 27 28 33 34 13 0 0 0 0 33 33 33 33 33 0 33 33 0 33 33 34 0 33 33 0 0 0 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2538 179 593 2060 11243 1898 1 2001 232 2538 179 593 2060 11243 1898 0 6 1265 3617 11155 1898 0 981 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;33 0 0 0 33 34 0 33 0 33 0 0 0 33 34 0 0 33 34 33 0 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7286 2305 323 80 3007 8 12600 7286 7413 1043 64 82 1257530 83 0;0 0 0 0 0 0 0 23 24 24 0 0 0 0 0;33 34 34 0 33 0 33 34 33 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6054 5611 182 116;0 0 0 0;33 21 22 33;1 1 1 1; -;43435 0 513 0 152812 0 519 0 313 0 531410 0 36 0 32049 0 513 0 313 0 71560 0 718343 0 368 0 30810 0 20551 0 101892;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 0 0 33 0 33 0 0 0 0 0 21 0 33 0 33 0 55 0 0 0 33 0 31 0 33 0 13;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 89 70 5396 2308 73 13379 8 4786 348 44768 10 8057 21 1 11826 2308 100 1 110 54 115 100;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 0 33 33 0 33 0 0 0 33 0 33 0 0 27 33 33 0 9 10 10 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1471 1480 1492 102 38 1266 2487 1080 213 196 171 2 1266 2487 1080 213 2 2452 562 5;0 9 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 13 13 33 34 33 33 34 34 33 33 0 33 33 34 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82610 0 30080 0 4276 0 154 7 57 62 77284 66 4884 40506 0 151 7 57 0 694 0 8706 10408 62 13 66 24602 150 86 19916 1 18361 150 86 1 128 5150 1 1196 1 2915 3002;0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 0 33 0 9 0 0 0 0 0 21 0 33 21 22 22 0 0 0 33 0 33 33 0 0 0 33 34 34 33 0 33 34 34 0 21 22 0 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2153 2238 12239 2238 980 129 195 25866 313 0 9906 19299 6628 40 0 622 2368 0 137 55 0 2 0 193 40 0 2 0 3099;45 46 46 46 46 0 0 0 19 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;53 54 54 54 54 0 0 33 25 26 26 27 33 33 0 0 0 0 21 22 0 0 0 21 22 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;85488 375 375 27847 2488 1 27847 3 18142 403 0 2 0 24051 1563 2204 5;21 0 0 21 0 0 21 0 0 0 0 0 0 1 2 2 2;31 0 0 33 0 0 33 0 33 33 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;331 489 68125 1631 2387 3081 11 162 29 1 4 9;0 0 0 23 24 24 0 0 0 0 1 2;0 11 12 29 30 30 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;7489 873 197 46 8455 4845 42 0 10552 3732 8951 0 2 0 31786 69 5;9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 33 33 34 0 33 0 33 0 0 0 13 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1073 1161 941 565 150 703 198 38 125 343 2 2 49 5355 941 565 48 102 38 1473 276 0;17 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 33 34 34 33 33 33 34 33 34 0 0 0 33 33 34 0 33 34 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;57 174626 59943 3 424 1755 2 45830 7 65 364 5;0 0 0 0 0 0 0 0 0 0 1 2;0 0 29 0 33 34 0 1 33 34 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;1844 5001 408 265 3176 377 36 8878 11026 177 377 177 57 265 228 6 377 15175 2970 3656 2758 408 19446 3176 377 36 377 14987 228 3656 265 128777 222 10212 239 3 9590 1929 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 33 0 0 0 0 0 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 0 0 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14735 813 51 21505 2773 1259 11902 7547 20095 194 1122 0 937 39 13 414 6893 1 6182 1 114 1044 980 1 114 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 17 0 0 0 1 2;0 0 0 33 34 34 0 0 33 0 33 0 0 33 34 33 33 0 47 0 31 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 7002 634 846 58 3 6150 9412 48 0 1030221 14 2006 0 1733 0 2362 16 941;27 28 28 28 28 28 28 28 28 0 21 0 0 0 0 0 0 0 0;0 35 36 36 36 36 36 36 0 0 27 0 33 0 33 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;20611 2702 6334 10680 58 2685 27348 24 1 4 209 132;0 37 38 38 0 0 37 0 0 17 23 24;33 45 46 46 0 33 45 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1; -;3145 20806 1 4 61;0 0 0 1 2;33 33 0 1 2;1 1 1 1 1; -;402 5754 2507 5109 63 2217 758 305 2287 17 3670 293 5109 8 373 305 2052 20 24 17 2 2507 280 382 0 14 2415 282 16;0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0;33 0 21 22 0 13 33 33 33 0 0 0 35 0 0 33 33 0 0 0 0 33 34 33 0 0 1 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 384 11453 11564 0 2908 2047 0 3378 3289 1778 0 496 1778 0 4304 1778 0 496 11453 87 16 37 6 87 6 44 6 3289 1778 6 363930 3221 62 4399 1172 66 2 395 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 33 0 0 0 35 0 33 0 33 0 33 33 0 33 33 0 33 33 31 0 33 0 31 0 33 0 0 33 0 13 0 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;280 120 647 2036 12245 1 2087 5;0 0 0 0 0 0 1 2;33 34 33 34 33 0 1 2;1 1 1 1 1 1 1 1; -;4004 629 46 1657 8596 13 145 145 1 44296 174 1 1655 3115 3168 5;0 0 0 0 0 0 0 0 0 21 22 0 1 2 2 2;33 33 0 33 34 34 34 34 0 27 28 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34 394 280 269 3261 15283 0 797 1041 327 30 1019 280 270 11178 29 1 131 2 7249 3545 1202 1 279 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 33 34 33 33 0 33 34 0 33 34 33 34 33 0 0 33 0 27 28 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;300 18161 5 0 2 0 187 300 18161 5;1 2 2 0 0 0 0 1 2 2;1 2 2 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1; -;5636 17172 30 1908 58 15 124 1 4 9;31 0 0 0 0 0 0 0 1 2;39 0 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;33 17022 1233 70 3933 1118 3 457 8 404 3 8 13646 22 214 7679 10 89 526 4880 21 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 0 33 34 0 33 0 33 34 0 33 0 0 33 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1362096 3 1088 18142 1 4 9;21 0 0 0 0 1 2;27 0 33 33 0 1 2;1 1 1 1 1 1 1; -;8751 3 34 3700 0 2 0 12806 13984;0 0 0 0 0 0 0 0 0;33 34 33 34 0 0 0 21 22;1 1 1 1 1 1 1 1 1; -;12376 614 2305 1176 8 293 233 1043 3 229 0 2 0 11 76 0 2 0 4904 0 2 0 2240 0 2696 0 709 197;31 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 1 2;39 0 0 33 0 0 33 34 0 33 0 0 0 33 34 0 0 0 31 0 0 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;23718 11212 117 422 2882 2134 753 1643 805 30 8336 3626 45 3 1591 2 14 1030 5 16 2 952 252 31 438 696 211 2 163 7 1058 7 65;37 38 0 0 37 0 0 0 0 0 37 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 0 0 33 34 34 33 34 0 45 33 33 34 33 0 0 1 2 0 0 33 0 33 33 33 33 0 33 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2619 12027 4267 5665 5765 16212 12 2723 2110 23640 2 2 452 2 2 169 5 0;17 18 18 0 0 0 0 0 13 14 0 0 0 0 0 1 2 0;31 32 32 33 33 33 33 34 17 18 0 0 33 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14860 54248 0 1249621 2 210 2 35 19 2 41 72;21 0 0 0 0 0 0 0 0 0 1 2;27 15 16 16 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;2157 2500 3288 485 34711 25981 1718 10123 0 54248 0 11695 0 148571 0 80765 1466 733 1 104 1 37 26 307 2 34 559 1466 5 0;0 0 0 11 12 0 0 11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2 0;33 34 34 25 26 33 33 15 0 21 0 33 0 25 26 26 33 34 0 33 0 33 0 33 0 1 2 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 14544 3 3038 2925 174 48 4963 4822 2208 1 41419 78;0 0 0 0 0 0 0 0 0 0 0 0 0;0 53 54 54 33 34 0 33 33 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;136 1069 820 1835 0 932 0 156670 0 2 0 433 1018 0 2 0 943 158;0 0 0 0 0 0 0 0 0 0 0 45 46 0 0 0 45 46;0 0 33 34 0 0 0 21 0 0 0 47 48 0 0 0 53 54;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1735 484 836 438 46 26597 0 139 39914 31713 12439 92521 2 5846 9722 276 144 1 2149 836 1 275 1503 1 1481 1 743 734 743 734;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;33 34 21 22 0 33 0 0 33 33 33 0 0 33 33 0 0 0 33 33 0 0 0 0 1 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;573 106 248 306 6199 13510 10 1867 12 663 1 13791 54 14516 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 1 2 2 2 0 1 2;33 34 33 34 33 0 0 0 0 33 0 1 2 2 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8236 996 9660 8106 444 12286 3525 74 2268 123 74 13678 1929 2 958 9660 8236 996 3525 2 41 996;17 18 9 9 10 0 0 0 0 0 0 0 0 0 0 0 17 18 0 0 0 0;31 32 13 0 0 33 33 0 33 33 0 0 33 0 13 14 31 32 33 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2792 148 2635 345 153 2 1674 245 70353 102 38 275 11 314 6215 352 561;17 18 18 18 18 0 0 0 9 0 0 0 0 0 0 0 0;13 14 33 34 34 0 33 33 13 33 34 33 34 34 34 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1651 623 5469 5296 1098 311 534 0 1651 623 5469 5296 1098 311 546 534 0 31 95 23 1367 25 0 2 0 11379 546 5;19 20 20 20 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 34 34 34 34 34 33 0 33 34 34 34 34 34 33 33 0 33 34 0 33 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;37279 13 1597 67 227 488 3086 25299 12012 0 2537 350 10 29 28771 36 37279 36 227 488 2 3291 1 5898 5 69;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 17 18 0;13 0 33 0 33 34 33 33 33 0 33 0 0 0 33 0 13 0 33 34 0 33 0 31 32 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2870 16 235 19 3592 374 8168 26 291 1 15506 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 34 33 33 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 27746 771 2689 1308 8597 3284 149 0 675 945 26471 0 87 2001 0 37 826 16 37 1 87 1 44 0 2 427 262 318 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 33 33 21 22 33 0 0 0 33 0 31 33 0 33 34 0 33 0 31 0 33 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 9118 3449 72244 16 9118 3449 72244 37 1 9118 3449 72244 104 1 9118 3449 72244 87 0 2 427 262 318 5;0 23 24 0 0 23 24 0 0 0 23 24 0 0 0 23 24 0 0 0 0 0 0 1 2;0 29 30 33 0 29 30 33 33 0 29 30 33 33 0 29 30 33 31 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 4883 472 17011 16 1 4883 472 17011 260 26 44 26 37 1 4883 472 17011 104 1 111;0 0 31 32 0 0 0 31 32 0 0 0 0 0 0 0 31 32 0 0 17;0 33 39 40 0 0 33 39 40 33 0 33 0 33 0 33 39 40 33 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;69;0;33;1; -;117 2 112794 2652 922 0 23 1134 1784 25 0;0 0 0 0 0 0 0 0 0 0 0;0 0 33 33 34 0 0 0 33 0 0;1 1 1 1 1 1 1 1 1 1 1; -;17870 2249 1623 416 1987 415 276 1 4 61;0 0 0 0 0 0 0 0 1 2;33 33 33 34 21 22 22 0 1 2;1 1 1 1 1 1 1 1 1 1; -;234 70 1832 22 8220 3709 4746 3709 43879 1148 2190 3 13 244 83055 141 478 20 32122 24 60 781 232 1 4 9;0 0 0 0 21 0 21 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 27 0 27 0 27 33 0 0 33 34 0 33 33 34 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;214 10 20 5459 17 1 4 9;0 0 0 0 0 0 1 2;33 34 34 0 0 0 1 2;1 1 1 1 1 1 1 1; -;2220 19049 9883 36 2220 19049 9883 18 36 19049 9883;17 9 0 0 17 18 0 0 0 9 0;31 13 33 0 31 13 33 33 0 13 33;1 1 1 1 1 1 1 1 1 1 1; -;51690 32790 9630 0 5332 1689 177 52 0 5332 1689 177 52 0 10833 9900 45077 2702 1 4 209 132;29 30 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;37 38 33 0 33 0 0 0 0 33 0 0 0 0 33 33 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 4450 21 4627 4644 1268 3 1511 9942 12 9 10 4644 1268 162 1 4 9;0 0 0 0 43 44 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 0 51 52 0 33 33 33 34 0 51 52 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;24318 148 559 2377 1583 120 153 950 337895 122 12205 15233 985 1812 6545 559 263 3125 148 0;17 18 18 18 18 18 18 9 10 10 10 0 0 0 0 0 0 0 0 0;13 14 31 32 32 32 32 13 14 33 34 34 33 33 33 33 34 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 140 5295 1399 170 40 16 36 140 5295 1399 40 1 1055 170 55 1 336 99 193 40;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 21 22 33 33 33 0 0 21 22 33 33 0 33 33 34 0 0 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16948 58 34306 36089 8231 655 1 16948 58 34306 36089 8231 44 40;39 40 0 0 0 0 0 39 40 0 0 0 0 0;33 34 33 34 33 33 0 33 34 33 34 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 139 67185 3651 144 898 4318 9583 12373 3 1757 48 14476 0 12212 18531 24470 62 5344 66 403 67 648 0 2 0 7121 5490 251 3651 6922 0 2 0 51318 235 197 0 2 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 34 0 0 33 34 34 0 33 0 33 0 0 0 0 0 33 0 33 0 33 0 0 0 0 33 0 33 0 0 0 0 0 33 34 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;806 78 49 8306 3 7013 48 78 10152 144 139 1486 2 78 2 35 19 2 41 72;0 0 45 46 46 46 46 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 33 34 34 34 34 33 33 0 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5044 945 194 3716 2 255 2 35 19 2 41 72;39 40 40 40 0 0 0 0 0 0 1 2;47 48 48 48 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;49 139 15905 144 94693 0 12185 0 46331 26 3116 195 173 181 48 765315 1 14 10714 301 36 4128 301 16 1 3820 1635 258;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 28 0 0 0 0 0;0 0 33 0 33 0 21 22 22 0 33 34 34 34 0 0 0 0 33 33 0 35 36 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8836 7546 2729 30 2568 626 56 3 6587 84 1 168 398 5;0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 43 44 0 33 34 0 0 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7660 62 7368 66 0 2 0 775 210 8 801 453 33;0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 9 0 0 0 0 33 33 0 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1; -;6625 7 0 15880 12047 172 10420 1964 850 106 625 68081 8 4689 40706 172 654 20912 2 247 934;0 0 0 17 18 0 0 0 0 0 0 0 0 0 9 0 0 0 0 17 0;33 0 0 31 32 33 0 0 0 33 34 0 0 33 13 33 33 33 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;739 241 4 493 4853 2788 1 8180 21898 1 114 204;0 0 1 2 0 0 0 0 0 0 1 2;0 0 1 2 33 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;31078 3851 12 1253 11074 30924 110 4019 870 1753 6906 8816 14 110 855 5 16;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0;33 33 34 34 33 13 33 0 0 0 33 33 0 1 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;45293 148 2635 153 0;17 18 18 18 0;13 14 33 34 0;1 1 1 1 1; -;49 3439 1836 10 250 14770 3 48 0 2 0 364 5;0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 33 34 34 34 34 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;330 42064 115 837 409 28 18 102 137 116 71 1 123 1 147 0 2 0 489 2182 5;17 18 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 1 2 2;13 0 0 21 22 33 34 33 21 33 34 0 33 0 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1973 814 2125 3 1897 6 814 2125 424 398 6 814 2125 2190 254 3 205 354 1 4462 725 2869 13176 1252 3057;0 21 22 0 0 0 21 22 0 0 0 21 22 0 0 0 0 0 0 9 10 10 47 48 0;33 27 28 0 33 0 27 28 33 33 0 27 28 0 0 0 33 34 0 0 0 0 41 42 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;139 818 144 127 8953 15016 792 1 4005 16110;0 0 0 0 0 0 0 0 0 0;0 33 0 0 0 33 33 0 33 33;1 1 1 1 1 1 1 1 1 1; -;1277 145 14678 2540 3 185 1 185;0 0 0 0 0 0 0 33;33 34 33 33 0 1 0 1;1 1 1 1 1 1 1 1; -;14 40488 142 6306 101 2286 778 446 16 1 40488 142 310 446 16841 327 1 4 493 210 150 446;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;0 27 21 22 22 33 0 33 0 0 27 33 34 33 0 0 0 1 2 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;617 77 1346 3841 988 3 248 494 15729 0 39 13 4981 12 10 3295 10673 2339;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 22 22;13 0 0 33 33 0 33 34 0 0 33 34 33 33 34 27 28 28;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7833 3040 2167 21 8 397 8243 236 32 7429 2653 8 1570 24 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 33 0 0 33 33 0 0 0 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;317156 62 88 66 383 28 18 0 2 0 1755;17 18 18 18 18 18 18 0 0 0 0;0 0 13 0 33 33 34 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1; -;73406 11347 2 73406 11347 44 2 933 1575 167 2 588 648 5;0 0 0 0 0 0 0 9 0 0 0 1 2 2;33 33 0 33 33 33 0 13 21 22 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12840 82 207 98 292 83 2109 82 106 29 1663 83 23 68 25 1 188 69;0 0 17 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 31 32 0 0 33 0 21 22 33 0 0 0 0 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 17487 267 87 16 1 410 17487 267 87 1 17487 267 87 104 26 2001 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;0 33 33 31 0 0 33 33 33 31 0 33 33 31 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3969 56 240 6618 21 15 143 17 250 1082 33 17 1 4 79 9;0 0 0 37 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 33 0 33 34 34 0 33 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;656 113 39 312 691 82 56461 1063 83 2997 1459 563 15287 3616 1 41644 2269 2 34 296 401 128 272 1068;9 10 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 9 0 0 0 0 0;13 14 33 34 0 0 33 0 0 47 48 0 33 13 0 21 22 0 13 33 33 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3939 367 41061 4874 416 1 4 61;0 0 0 0 0 0 1 2;33 34 0 33 33 0 1 2;1 1 1 1 1 1 1 1; -;49 9488 8634 160 1963 48 9880 0 2691 475 6384 8590 10 992 1 188 634;19 20 20 20 20 20 0 0 0 0 0 0 0 0 0 0 0;0 25 26 26 26 0 33 0 0 0 33 33 0 0 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 1409 13584 30 22 40087 53168 23727 3 8 1933 455 4811 22 203 8 33055 10874 134 34769 63 8 8564 7537 21 1 4 9;0 0 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 37 0 0 21 33 34 0 0 0 0 33 34 0 0 0 33 0 0 0 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;102 38 218 190 2813 12508 1192 81 1260 6 190 2813 12508 1192 11 91 173 12508 1192 15 6779;0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 13 0 33 21 22 0 33 0 0 33 21 22 21 22 22 21 22 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;38275 376 36 46795 75 38275 376 10171 12156 1311 406 75 1720 297 125 834 1190;9 10 0 9 10 9 10 0 0 0 0 0 0 0 0 0 0;13 14 0 13 14 13 14 33 33 33 33 0 0 33 34 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14704 83521 106061 3 613 32 4161 10 20 60 1 4 130 112;21 0 0 0 0 0 0 0 0 0 0 1 2 2;27 33 34 0 33 0 33 33 34 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2934 5445 284 7846 1136 3436 6892 21 652 6892 22256 4652 278 21 1709 11 64 3 746 90 373 9 256 1 4 9;29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 0 33 33 33 0 33 34 0 33 33 0 0 33 33 34 0 33 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 22 166 22 1565 404 3 89 11 4659 21 8 3414 1476 457 7 7 7 0 1685 9421 4237 2127 0 2 0 185;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0;0 21 22 22 33 33 34 33 34 33 0 0 33 34 33 0 0 0 0 33 13 33 0 0 0 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;46345 67 12581 17523 1394 0 1999 6933 1787 1 34 389 5 0;21 0 0 0 0 0 0 0 0 0 1 2 2 0;27 0 27 33 33 0 33 33 34 0 1 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4741 15954 628 197 120 1056 58;17 18 18 18 18 18 18;13 33 0 33 34 33 34;1 1 1 1 1 1 1; -;11802 1186 3 1080 10 91 17 1 105 412 1266 2 412 2523 112 197;17 0 0 0 0 0 0 0 0 9 0 0 9 5 6 0;31 33 0 33 0 0 0 0 33 13 33 0 13 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2933 12 9 341 21 20 946 501 0 541 773 1783 1636 0 781 1047 1 4 9;0 0 0 0 0 0 37 38 0 0 0 0 0 0 0 0 0 1 2;33 33 34 33 34 0 33 34 0 0 33 33 33 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12644 321 298 3 8069 67 17569 1260 0 2 0 364 5;0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 0 0 33 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;4384 1900 33608 2406 737 1234 42 3928 158 463 5 1668 2 2 169 5 466 9600 2 2 169 5 0;17 0 0 0 0 0 0 0 39 40 40 40 0 0 0 0 0 0 0 0 1 2 0;31 21 22 22 29 30 30 33 47 48 48 48 0 0 1 2 2 2 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;48141 3 226 1 48141 273 1 48141 140 15 165 1 704 731 1538;17 0 0 0 17 0 0 17 0 0 0 0 5 6 6;31 0 33 0 31 33 0 31 33 34 34 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;768 3 13 29 8655 8 317 457 85 29 97 284 106 324 2224 63 13 324 36 984 554 2 25443 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 17 18;33 25 26 26 33 0 0 33 0 0 0 0 33 34 0 0 33 34 0 13 33 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;28392 0 60320 0 85 0 49 8722 12292 85 48 0 17038 0 2 0 31413 190 462 0 139 72341 144 53 31 291 53 387 5 8 19 35 31 95;0 0 0 0 0 0 45 46 46 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;53 54 54 0 0 0 53 54 54 54 54 0 33 0 0 0 33 33 34 0 0 33 0 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;179200;21;33;1; -;14395 0 1604 20817 0 16700 15 303 3502 203 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;31 0 0 0 0 29 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;24031 140 406 2783 228 327 3816 1 279 50 5077 5 1 279 5;17 18 0 0 0 0 0 0 1 2 2 2 0 1 2;31 32 33 33 0 0 0 0 1 2 2 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;960 42 147026 395271 7559 7992 1084 1 147026 35164 1 33086 44;0 0 21 0 0 0 0 0 0 0 0 0 0;33 34 27 0 33 33 34 0 27 21 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;407 3 5681 483 1576 176 156 2 5681 483 3493;0 0 0 0 0 0 0 0 0 0 0;0 0 33 34 33 0 33 0 33 34 33;1 1 1 1 1 1 1 1 1 1 1; -;895 113 172 3152 2813 102 2797 719 20 356 166 1222 1 4 9;9 10 0 0 0 0 0 0 0 0 0 0 0 1 2;13 14 33 33 33 33 0 21 22 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7924 27 1161 345 516 5;1 2 2 2 2 2;13 14 1 2 2 2;1 1 1 1 1 1; -;62 4043 2678 4243 75 66 1482 8 5316 45232 45 8 15073 265 22654 8 568 3587 121 16506 3 1859 4160 4066 3 9238 715 8 2758 1903 1009 15073 74 22654 600 145 1689 74 2908 388 62 57 66 9892 46 4066 10 11017 3587 3 19218 2059 62 52 66 1442 5316;0 0 0 9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 0 13 14 0 0 0 0 33 0 0 33 0 33 0 0 0 0 33 0 0 0 33 0 33 0 0 0 33 0 33 0 33 0 0 0 0 0 0 0 0 0 33 0 33 0 0 0 0 33 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2658 3013 5595 89 1688 371 0 516 231 3222 385 32 9884 1 78 1 279 5;45 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;53 54 54 0 21 22 0 33 33 34 34 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4501 6805 56 6556 5 147 91 36 4501 6805 56 6556 5 147 91 1 14 334 1641 430 988 3567 16;0 0 0 1 2 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0;51 33 0 1 2 33 0 0 51 33 0 1 2 33 0 0 0 33 33 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1013 6709 54630 0 26309 289 12706 8978 2 5501 2 1446 6995 268 255 5;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 0 33 0 33 0 33 33 0 33 0 33 34 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;933 119 257 268 257 3 185 1 185;17 18 18 18 18 0 1 0 1;31 32 32 21 33 0 1 0 1;1 1 1 1 1 1 1 1 1; -;2882 2581 3 3031 1231 12 4673 16543 2 18398 1634 5;37 0 0 0 0 0 0 0 0 1 2 2;33 33 0 33 34 33 34 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1; -;16785 32 2017 2 315 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 1 2;33 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;14 227 299 7022 228908 1689 26 688 300 921 2062 776 242 16 37 6 87 6 44 6 20559 294 732 6 461 113 157 27 5190 27889 2057 3312 28 18 2 395 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 10 9 10 0 17 18 18 18 18 0 1 2;0 0 33 37 0 0 0 0 33 33 33 33 34 0 33 0 31 0 33 0 33 34 34 0 13 14 13 14 0 17 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2432 50 704 3 1171 39 431 224 1289 403 0 123315 3 39 13 29 2233 8085 7042 1394 1036 167 131 2 2950 1833 5;19 20 20 20 20 0 0 0 0 0 0 21 0 0 0 0 19 20 0 0 0 0 0 0 1 2 2;25 26 26 26 26 0 33 0 33 33 0 21 0 33 34 34 33 34 33 33 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1552 1833 122 749 547 491 16266;27 28 27 28 28 28 0;35 36 33 34 34 34 33;1 1 1 1 1 1 1; -;14 277046 19568 16 37 6 87 6 44 6 2905 1765 26 8327 6 5060 32689 62 4399 1172 66 2 395 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 22 0 0 0 0 0 1 2;0 0 33 0 33 0 31 0 33 0 33 34 0 31 0 27 28 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4061 501 2685 21870 12 2477 8 595 33701 1 11 217 846 1021 1 595 217 132;37 38 0 37 38 38 0 37 38 0 0 0 0 0 0 0 0 0;45 46 33 45 46 46 0 33 45 0 1 2 2 2 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;282768 0 116323 0 37191 0 519 0 1416869 0 2 0 4779 0 308456 0 13153 0 36570 0 28319 0 2097 0 36454 53933;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 0 33 0 37 0 33 0 0 0 0 0 21 0 0 0 33 0 33 0 51 0 33 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8143 574 181 46 7436 3685 7591 8184 7467 10597 263 6876;0 0 0 0 17 18 0 0 0 0 0 0;33 33 34 0 31 32 33 33 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1; -;14 74450 1659 212 16 137 1466 36 741 36 87 36 104 36 37 36 123 36 3931 1 65813 2353 5;0 13 14 14 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18;0 17 18 18 0 21 33 0 33 0 31 0 33 0 33 0 33 0 33 0 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;81 2425 14219 81 1252 1941 15 124 17 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;33 34 31 13 14 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;6488 39136 4758 1 6488 39136 570 1 6488 39136 570 372 1 6488 39136 4620 3126 1 6488 39136 24159 3126 1 6488 39136 4758 741 42 1 43563 25988 0;9 0 0 0 9 0 0 0 9 0 0 0 0 9 0 0 0 0 9 0 0 0 0 9 0 0 0 0 0 21 22 0;13 13 33 0 13 13 33 0 13 13 33 33 0 13 13 33 34 0 13 13 33 34 0 13 13 33 33 34 0 27 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 397 5168 8 96 10 236 397 160 1419 1118 8 135 10 817 1015 8 373 32929 24 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;55 56 33 0 0 0 0 33 0 33 34 0 0 0 0 33 0 35 36 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;62 57 66 1482 3003 8 7680 1626 74 2602 2137 1626 134 10 2135 1 1 1 1 1 1 3 4072 1204 289 3 2059 68 45 7680 1626 3 24055 10 1 1 1 1 1 1 8 1442 3229 64 2114 7680 1626 2657 1 361 1 4 554;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 0 0 33 34 0 33 33 34 0 0 33 0 0 0 0 0 0 0 33 0 0 0 0 0 0 33 34 0 33 0 0 0 0 0 0 0 0 0 33 0 0 33 34 33 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;68165 900 527 1649 0 4446 760 61284 1 69 1 192 491 1 2220 1 2220 1006 1 19135 4053 1 2220 10783 1896 414;29 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 27 28 0 27 28 0 17 18 18 18;37 33 34 0 0 33 0 33 0 33 0 33 34 0 31 0 35 36 0 33 34 0 31 32 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2868 2426 383 0 2 0 6452 929 18 0 2 0 33 227 6452 929 5;0 0 0 0 0 0 17 18 18 0 0 0 1 2 2 2 2;0 0 33 0 0 0 31 32 32 0 0 0 1 2 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;399 46 317 1412 11 5586 82 1819 1742 83 1 678 287 2340 5283 1 34 287 5;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;13 0 0 33 0 33 0 21 22 0 0 33 34 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;635 3057 403 8 635 626 1088 761;9 0 0 0 0 0 0 0;13 33 33 0 33 34 33 33;1 1 1 1 1 1 1 1; -;9552 1896 0 349 4213 2 0 17107 465 0 2 0 19 709 465 211;17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0;31 32 0 33 33 0 0 33 33 0 0 0 33 34 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;24520 699148 2 797 6867 224 26063 1835 6004 403 24826 29548 4764 26146 54 913 23023 1 797 6867 4405 1 114 204;0 0 0 0 0 0 0 0 0 0 0 9 10 10 0 0 0 0 0 0 0 0 1 2;0 0 0 0 0 0 0 33 33 33 0 13 14 14 0 0 33 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 389 16554 6626 1818 565 63 178 1506 446 16 1 389 16554 6626 1818 565 63 178 1506 1 4 493;0 11 12 11 12 0 0 0 0 0 0 0 11 12 11 12 0 0 0 0 0 1 2;0 15 16 15 16 0 21 22 0 33 0 0 15 16 15 16 0 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;389 113 1317 314 159 28 18 3317 199;17 18 18 18 18 18 18 0 0;13 14 33 33 34 33 34 33 34;1 1 1 1 1 1 1 1 1; -;1942561 3 424 398 0 2 0 1689 5878 456 5 0 2 0 34 77 35765 456 11576 67 3096 456 230 3 138 235 232;0 0 0 0 0 0 0 1 2 2 2 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 33 0 0 0 1 2 2 2 0 0 0 13 0 33 33 33 0 33 33 34 0 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;218 7909 1773 284 5549 8973 9284 0 5037 46 1275 4594 11 6008 1282 2 247 69;9 0 0 0 0 0 0 0 0 0 0 0 0 37 38 0 0 0;13 33 33 0 1 33 34 0 0 0 0 0 0 33 34 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;402 15 309 3547 765 25069 3 578 1375 0 2 0 896 14127 220 229 6082 0 8372 1875 0;0 0 0 0 0 0 0 0 0 0 0 0 23 24 24 0 0 0 17 18 0;33 0 0 33 33 0 0 33 33 0 0 0 29 30 30 33 33 0 31 32 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;37840 438 224 0 25134 0;0 0 0 0 0 0;21 22 0 0 33 0;1 1 1 1 1 1; -;650 823 23730 107 33;9 0 11 0 0;13 0 15 33 0;1 1 1 1 1; -;28887 23637 1 28887 23637 37 1 410 28887 23637 104 26 182 0 2 0 111;31 0 0 31 0 0 0 0 31 0 0 0 0 0 0 0 17;39 33 0 39 33 33 0 33 39 33 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;20802 7700 31662 1392 294 0 2 0 5394 5619 1196;17 0 17 0 0 0 0 0 1 2 2;31 47 31 33 34 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1; -;16690 2162 40 12 21 21 8 10 22 385 24 0 2 0 6901 773 1402 0 2 0 6901 43;5 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;9 10 33 33 34 0 0 33 34 0 0 0 0 0 33 21 22 0 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2350 90 1001 89 3975 15217 8 1534 118 90 21387 15217 32 659 6365 8 6234 134 10 2297 4264 8 90 166 219 607 11 915 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 0 33 33 0 9 10 0 33 33 0 0 33 0 33 0 0 33 33 0 0 0 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;35670 0 21056;0 0 0;33 0 13;1 1 1; -;38199 794 3426 202 58 46 7973 1297 1449 527 0 2 0 294 552 69;0 39 40 40 40 0 0 0 0 0 0 0 0 0 0 0;31 47 48 48 48 0 33 0 33 0 0 0 0 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1327 16 3162 3425 54 667 417 1309 0 37526 0 529 2988 4343 59 62 46003 125082 66 0 170 31 367 1 8566 301 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 33 0 35 36 0 35 36 36 0 0 0 21 22 22 0 0 33 0 0 0 33 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6868 3 185 1 279 185;0 0 0 0 1 2;25 0 1 0 1 2;1 1 1 1 1 1; -;11192 15539 3517 0 2690 826 77 179 7419 1891 1 114 110 1 114 5;7 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;11 33 33 0 33 33 33 34 0 33 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12848 207 3128 703 3097 30662 211 261 2462 261 2 34 495 71 5;0 0 0 29 0 17 0 0 0 0 0 1 2 2 2;33 33 33 33 0 31 33 33 33 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2476 257 1199 417 1993 174 304 1668 1213 2878;17 18 0 0 0 0 0 0 0 0;21 22 33 0 33 34 33 33 33 33;1 1 1 1 1 1 1 1 1 1; -;615 325 2098 55 85 97 78607 2179 2 247 325;0 0 0 0 0 0 0 0 0 0 0;33 34 34 34 33 34 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1; -;167 1 18032 7 65 16773 5;0 0 0 0 0 1 2;33 0 0 33 34 1 2;1 1 1 1 1 1 1; -;1493647 0 123 4009 196 36 94 730 3630 0 2 0 6623 43;0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 34 33 0 33 34 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2414 3 350 603 135 1009 29 462 6058 1 23446 119 512 257 43 1 4 47;0 0 0 0 0 0 0 0 0 0 17 18 18 18 0 0 1 2;33 0 0 0 0 33 34 34 0 0 31 32 32 32 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1045 0 10392 0 13024 32609 0 4917 149 303 3781 6319 1 1109 11687 5;5 6 6 0 0 0 0 0 0 0 0 0 0 0 1 0;9 10 10 0 0 33 0 31 33 33 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1200 16160 459 212 8 126 2731 848 1117 432 17 214 10 15 441 17 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 0 0 0 21 0 0 0 0 33 34 34 34 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;25074 16056 3312 787 15943 189 5359 3 10 20 1 4 9;7 8 0 0 0 0 0 0 0 0 0 1 2;11 12 33 34 33 0 33 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;684 561 0 2 0 19920 148 287 411 5;0 0 0 0 0 0 0 0 0 0;33 33 0 0 0 13 14 33 33 34;1 1 1 1 1 1 1 1 1 1; -;1874 113 251 12836 27 251 69002 148 251 2214 1446 376 0 6442 8 69002 148 62206 4902 3344 369 4902 3 147 0 2 0 88 469;9 10 0 9 10 0 9 10 0 9 10 10 0 0 0 17 18 18 18 18 18 18 0 0 0 0 0 0 0;13 14 0 13 14 0 13 14 0 13 14 14 0 33 0 13 14 31 33 33 34 33 0 33 0 0 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4885 3 1459 654 2 2 2 5827 2464 1 19 31 95 0 2 0 1249 7 65;0 0 0 0 0 0 0 39 40 0 1 2 0 0 0 0 1 2 2;33 0 33 34 0 0 0 47 48 0 1 2 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 49 26401 30 56 3 6736 4326 0 13325 48 0 32 49 22 3 15722 10 5111 0 13325 48 0 87458 0 3 483 0 781 0 869 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 34 34 0 0 33 0 33 0 0 0 0 35 36 36 0 33 0 33 0 0 21 0 0 0 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; diff --git a/PaddleNLP/unarchived/chinese_ner/data/train_files/train_part_2 b/PaddleNLP/unarchived/chinese_ner/data/train_files/train_part_2 deleted file mode 100644 index 63077b6b11c70c404201ed703011376efb99eac1..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/data/train_files/train_part_2 +++ /dev/null @@ -1,500 +0,0 @@ -;3194 11 21067 1 761 5 382;0 0 0 0 1 2 0;33 0 33 0 1 2 33;1 1 1 1 1 1 1; -;1942561 21763 0 43370 0;0 0 0 0 0;0 21 0 0 0;1 1 1 1 1; -;45844 0 473 0 7200 0 6535 2 105746 0 460582 6 45844 0 473 0 7200 0 6535 31 1748 6 1889 170 40 6 45844 0 473 0 7200 0 6535 579 40 1 4 202 2 4385 5522;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6 0 0 0;21 22 22 22 22 22 22 0 27 28 28 0 21 22 22 22 22 22 22 33 0 0 9 33 33 0 21 22 22 22 22 22 22 33 33 0 33 34 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;849 110 10927 302 7166 46 1285 311 12 4321 227 8 4897 12 7152 1 463 5 2325 2 1533 31;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 0 33 0 33 34 35 36 0 0 33 0 33 0 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 27 1916 24483 548 407 15763 58124;0 0 0 0 0 0 0 0;0 0 33 33 33 0 33 33;1 1 1 1 1 1 1 1; -;14 0 8926 736 37 16 8926 736 87 36 8926 736 260 36 8926 736 44 0 2 0 34 997 516 5 0;0 0 29 30 0 0 29 30 0 0 29 30 0 0 29 30 0 0 0 0 1 2 2 2 0;0 0 37 38 33 0 37 38 31 0 37 38 33 0 37 38 33 0 0 0 1 2 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2230 1980 1425 4304 104 0 1764 5557 1521 9069 12233 16073 6438 323824 3289 4930 16 37 6 87 6 44 6 3289 4930 6 92 2780 1693 1012 28 18 2 395 5;0 0 9 0 0 0 0 0 0 0 0 0 0 0 43 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 0 1 2;0 33 33 33 33 33 0 0 33 33 0 33 33 33 51 0 33 0 33 0 31 0 33 0 0 33 0 13 0 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15 40 12 21 2045 1119 39911 1 4 9;0 0 0 0 0 0 0 0 1 2;0 33 33 34 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1; -;2793 0 2 0 280396 14358 0 15520;0 0 0 0 0 0 0 0;33 0 0 0 0 33 0 33;1 1 1 1 1 1 1 1; -;902 5500 30 157 21521 903 91 180 17 2 8411 1331 2917 5;0 37 0 0 0 0 0 0 0 0 1 2 2 2;0 45 0 13 14 0 0 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9829 0 1618 1116 3 118844 59 2 401 2 35 19 2 41 72;17 0 0 0 0 21 0 0 0 0 0 0 0 1 2;31 0 33 34 0 27 0 0 33 34 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;576 5037 931 24551 3730 5726 753 2 45 81 5;9 0 0 0 0 0 0 0 1 2 2;21 0 0 33 34 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1; -;1011 3780 6260 3 314 67 120 1 4 61;0 0 0 0 0 0 0 0 1 2;33 33 33 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1; -;571 2612 35829 161 1858 9686 31 3399 243 2752 2 2 49 198 38 34 544 663 3674 2240 4966 609 224 62 39 13 1134 66 48 198 38;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 28 28 28 28 28 28 28 0 0 0 0 0 0 0 0;33 33 33 33 33 33 33 34 33 33 0 0 0 33 34 35 36 36 36 36 36 36 36 0 33 34 34 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;90 1442 17880 2 11195 97 3251 6 90 1442 17880 1288 170 40 6 35 1288 31 95 0 1 4 202 2 4385 5522;33 34 34 0 21 22 22 0 33 34 34 0 0 0 0 0 0 19 20 0 0 5 6 0 0 0;33 34 34 0 27 28 28 0 33 34 34 33 33 33 0 33 33 33 34 0 0 33 34 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;459 283 29 97 1369 7132 11 356 3488 10 15 441 60 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 0 33 33 33 34 33 0 35 36 36 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;17391 32 34766 15 64 203 17 0 1 2375 499 382;13 0 13 0 0 0 0 0 0 0 0 0;17 0 17 0 0 0 0 0 0 21 22 33;1 1 1 1 1 1 1 1 1 1 1 1; -;82 73576 0 55434 83 0 513 0 313 0 178469 0 25093 0 17993 0 158306 0 290093 0 2 0 1066 0 5578 0 59670;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 0 0 0 0 0 0 0 0 0 0;0 21 0 33 0 0 33 0 33 0 0 0 55 56 56 0 21 0 55 0 0 0 35 36 36 36 36;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 4241 64 685 86 16 4241 64 685 86 37 1 4241 64 685 86 104 1 4241 64 685 86 87 0 2 427 262 318 5;0 0 0 0 0 0 0 0 23 24 0 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 29 30 0 33 0 29 30 33 0 33 0 29 30 33 0 33 0 29 30 31 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2920 1942561 3 47;0 0 0 0;0 0 0 1;1 1 1 1; -;12064 27 184 12487 102 137 116 71 1 123 1 147 0 2 0 489 2182 5;17 18 18 18 0 0 0 0 0 0 0 0 0 0 0 1 2 2;13 14 33 34 33 21 33 34 0 33 0 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2200 7068 8 786 11 20 415 93 155 1 4 209 132;37 38 0 0 0 0 0 0 0 0 17 23 24;45 46 0 33 33 34 0 33 34 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1; -;543 1264 686 2 543 23 207 25 1264 686 28 18;0 0 0 0 9 0 0 0 0 0 0 0;13 33 34 0 13 0 33 0 33 34 33 34;1 1 1 1 1 1 1 1 1 1 1 1; -;139 4007 144 1642 140 39 312 917 387229 11518 0 2 0 364 5;0 29 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 34 33 34 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14444 46854 3088 10733 34 159 379 10732 1 34 4306 852 98 43 1 4 47;0 21 0 0 0 0 0 0 0 1 2 2 2 2 0 1 2;33 27 33 33 21 22 22 22 0 1 2 2 2 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3233 4330 1391 308 1218 1220 51 1315 580 308 577 194 555 1688 502 10223 1 268 1 279 5;9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 14 0 0 33 33 0 0 0 0 33 34 33 34 0 33 0 21 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 82156 0 16406 0 32 15696 3 1464 781 1 4 9;0 0 0 0 0 0 29 0 29 0 0 1 2;0 27 0 0 0 0 37 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;14227 33 8951 735 1 188 69 108;0 0 0 0 0 17 18 18;55 0 33 0 0 9 10 33;1 1 1 1 1 1 1 1; -;4492 332062 46 48428 3 2399 174 62 371 583 34 251 2369 560 251 198 861 311 66 1 188 268;21 22 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 23 24;0 0 0 21 0 33 34 0 33 34 13 0 21 22 0 33 33 34 0 0 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;64 5566 1665 109 484 11453 8 710 109 14773 644 2522 146 590 8 710 109 51517 644 4775 52 590 8 1255 91 124593 17 1 4 9 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 1 2 0;0 0 0 0 33 33 0 0 0 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 15 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;19373 1589 41342 6 19373 41342 6 19373 1800 414 6 19373 1800 414 28 18;0 0 0 0 0 0 0 17 18 18 0 17 18 18 18 18;33 21 22 0 33 33 0 31 32 32 0 31 32 32 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;490891 2 2129 518 7980 21349 76 2945 1225 31466 267 4380 1523 507 161 3 208 1 4 61;0 0 0 0 23 24 24 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 0 29 30 30 0 33 33 33 33 0 33 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;157 444 13 1861 1505 192 119 28 18;17 18 18 18 18 18 18 18 18;13 0 33 34 0 33 34 33 34;1 1 1 1 1 1 1 1 1; -;227 20 6193;0 0 11;0 0 33;1 1 1; -;13 29 1976 63 21 1318 3 1215 3011 8 126 373 12 70 166 30840 15025 8 3802 2553 341 1361 24225 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 33 34 33 0 33 34 0 0 0 33 34 0 33 33 0 33 33 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;35445 21 13 4840 41360 358 8 2002 7138 888 502 951 9202 8 999 15101 1882 1882 3 8 12 10 23182 21 8 15 4563 17 1 4 9 0;0 0 0 0 29 30 0 29 30 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;33 33 34 0 37 38 0 37 38 0 0 0 33 0 0 33 33 34 0 0 33 34 0 0 0 0 0 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4902 2361 31278 125 508 2 6194 61;0 0 0 0 0 0 0 0;33 33 33 33 34 0 33 33;1 1 1 1 1 1 1 1; -;8935 3 669 1 4 61;0 0 0 0 1 2;33 0 33 0 1 2;1 1 1 1 1 1; -;43232 3218 28 18;17 18 18 18;33 33 33 34;1 1 1 1; -;14 9666 4749 110 4451 178 147 36 123 36 826 16 9666 4749 110 4451 178 137 307 1 961 110 5;0 0 9 10 10 10 10 0 0 0 0 0 0 17 18 0 0 0 0 0 1 2 2;0 13 31 32 33 34 33 0 33 0 33 0 13 31 32 33 34 21 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;236 30 18901 227 29 1495 8 18901 15 124 1 4 9;0 0 9 0 0 0 0 9 0 0 0 1 2;0 0 13 0 0 33 0 13 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 536 6990 1523 482 8153 0 536 19337 0 6837 12174 191 64 6990 1523 16 37 6 87 6 44 6 1208 1165 270 1148 6 9922 2284 75 103078 12165 267 1172 244 2 395 5;0 9 31 32 23 24 0 0 0 0 0 0 0 0 31 32 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 18 18 0 1 2;0 13 39 40 29 30 0 13 33 0 21 22 22 0 39 40 0 33 0 31 0 33 0 33 34 33 33 0 13 33 34 33 33 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;28884 2 343673 0 566 0 52380 0 22342 0;0 0 0 0 0 0 0 0 0 0;33 0 0 0 0 0 35 36 36 0;1 1 1 1 1 1 1 1 1 1; -;18255 297 286 1975 508 0 2 0 364 5;0 0 0 0 0 0 0 0 1 2;33 34 33 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;540 3127 21999 37 189 14 68 16 1 2818 40642 2869 2828 1 5274 1242 344 2788;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 33 0 0 0 0 0 0 33 0 0 0 21 22 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4931 276 2283 58 5189 249 79 24 17 1 4 79 9;37 38 38 0 0 0 0 0 0 0 0 0 0;33 34 33 0 33 0 21 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1; -;299766 43720 0 897 1807 0 177 14843 2 22880 5 897 100 0;0 0 0 0 0 0 0 0 0 1 2 0 0 0;0 33 0 33 33 0 0 29 0 1 2 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1108 9866 58460 1335 1714 370 23 68 25 2 52032 5049 1679 2 92 2 588 648 5;11 12 12 0 17 18 0 0 0 0 0 0 0 0 9 0 1 2 2;15 16 16 0 31 32 0 0 0 0 21 22 22 0 13 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1128 56 214 117 390 101 0 90 12802 10 32 335 11273 15105 1318 2 7698 5;0 0 0 0 0 0 0 0 0 0 0 21 22 27 28 0 1 2;33 34 0 0 0 0 0 0 35 36 0 27 28 35 36 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;25067 768 3 44503 4395 39 52 224 2 205 354 2 35 979 19 2 41 72;19 20 20 20 20 0 0 0 0 0 0 0 0 0 0 0 1 2;25 26 26 26 26 0 0 0 0 33 34 0 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 340 80939 174039 1269 345 150 28 18 719 36 340 80939 174039 1269 345 150 28 18 1183 15 124 16 2 136 1448 5;0 17 18 18 18 18 18 18 18 0 0 17 18 18 18 18 18 18 18 0 0 0 0 0 1 2 2;0 13 31 0 33 34 34 33 34 33 0 13 31 0 33 34 34 33 34 33 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 1232 1731 2289 52 48 7462 39 248 604 231 971 3519 282 1 557 3355 5;45 46 46 46 46 46 9 0 0 0 0 17 0 0 0 1 2 2;53 54 54 54 54 54 13 21 22 0 0 31 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 609 16 49 207 263 67 544 195 4211 323 126412 15659 23 13562 25 48 166 22 275 3 506 176 4998 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 0 33 34 0 33 33 34 34 33 33 0 33 0 0 0 21 22 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;923 113 4270 11744 21713 409 153;17 18 18 18 18 18 18;31 32 32 32 32 32 32;1 1 1 1 1 1 1; -;22450 20952 5801 219 87 33 0 566 0 337 1951 0 2 0 2244 5;9 9 10 0 0 0 0 0 0 0 0 0 0 0 1 2;13 13 14 0 31 0 0 0 0 33 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;44126 4975 1 577760 220 1795 52626 47131 1 525 7 6894 1 1115 40;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 0 21 21 22 27 33 0 0 0 33 0 9 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12 96 317 210 216 6184 1 4564 5;0 0 0 0 0 0 0 1 2;33 34 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1; -;6137 50 1963 40 1 695 94 1581 184 793 55 78 282 1 713 54 115;45 46 46 0 0 23 24 0 0 0 0 27 28 0 1 2 2;53 54 54 33 0 29 30 33 33 33 34 35 36 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;105 567 1950 13486 2095 416 6 13486 2095 692 2 726 5261 5 1311;0 0 0 0 0 0 0 0 0 0 0 1 2 2 0;33 33 34 34 34 34 0 33 34 33 0 1 2 2 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 12 10 1437 924 4724 134 58 11 7474 8 1160 8 1437 924 7474 134 1820 4724 8 214 10 13 29 12 9932 3 12 333 10164 3 9683 83 64 226 15 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 33 34 33 33 34 0 33 0 0 0 33 34 33 0 33 33 0 0 0 33 34 33 34 0 33 34 34 0 33 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;122548 0 50585 10 20 156 1 4 130 112;0 0 0 0 0 0 0 1 2 2;0 0 0 33 34 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1; -;911 1299 133 394 701 145 561 133 0 40468 937 2308 15677 1344 418 2 45 81 5;9 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 1 2 2;13 33 0 33 33 34 33 0 0 27 0 33 33 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5122 1217 99 2085 1458 1 330 5122 882 1217 99;0 0 0 0 0 0 0 0 0 0 0;21 33 34 33 33 0 33 34 33 33 34;1 1 1 1 1 1 1 1 1 1 1; -;15 827 2114 11263 0 820 1438 3 6132 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 15 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;53516 90 37419 21 84 90 160 799 43 84 331 22004 3357 1 53516 43 1 4 47;21 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 0 1 2;27 0 33 0 0 21 22 22 22 0 0 33 0 0 27 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3710 6 7184 766 31683 1 4 61;0 0 37 38 38 0 1 2;33 0 45 46 46 0 1 2;1 1 1 1 1 1 1 1; -;103 7679 1687 96 30 13 332 2 315 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 35 36 36 36 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49257 3338 11 256 2475 3 2582 432 17 1 4 9;29 30 0 0 0 0 0 0 0 0 1 2;37 38 0 0 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;933 27 4740 496 1023 108 319 5536 1523 12510 440 882 1 33 505 31;17 18 18 18 18 18 0 37 38 38 0 0 0 1 2 2;31 32 32 32 32 32 33 33 34 34 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;317 8817 10715 42552 3 634 1 188 69 108;0 0 0 0 0 0 0 17 18 18;0 33 34 33 0 33 0 9 10 33;1 1 1 1 1 1 1 1 1 1; -;102 617 1417 608 1149 1018 5417 0 39 13 407 4260 10 659 1 893 3591;0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 13 33 33 34 34 33 0 33 34 0 33 0 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;31509 148 5700 1363 1696 0 8335 20346 172 4635 4703 3234 1 50 4519 5 1 279 5;9 10 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 1 2;13 14 33 33 33 0 33 0 33 33 33 34 0 1 2 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;30953 336 7 129 303 229 17 0 2 0 651 751211 3 1487 0 2 0 605;0 0 0 0 0 0 0 0 0 0 21 22 0 0 0 0 0 1;9 0 0 0 33 33 0 0 0 0 0 0 0 33 0 0 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16597 174 229989 26972 742 0 882 1279 3018 6524 4170 452 229 2 247 69 108;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 7 33 0 0 33 0 33 27 33 33 34 0 9 10 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 4220 4794 288 21959 63 178 1506 446 16 1 4220 4794 288 21959 63 178 1506 1 4 493;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 33 33 34 21 22 0 33 0 0 33 33 33 34 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7186 27 49801 14472 207 1012 28 18;17 18 18 18 18 18 18 18;13 14 33 34 47 48 33 34;1 1 1 1 1 1 1 1; -;22 249 8822 18 216 123 8 2227 165 11 3056 3 17150 21 8 10 20 156 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 33 33 34 0 33 34 0 33 0 33 0 0 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;628 1239 573 18510 64062 0 2730 6356 82 7366 4198 83 0 1 235 78 69 1 1945 7 65 34 78 39 13 411 199;17 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0;31 32 32 33 33 0 33 33 0 33 34 0 0 0 33 34 33 0 1 33 34 13 33 33 34 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3875 41586 8048 3 24240 1 4 9;0 0 0 0 0 0 1 2;33 33 34 0 33 0 1 2;1 1 1 1 1 1 1 1; -;375 8772 62312 46 10161 5580 2470 8168 1219 1680 0 154 181 18112 1 375 8772 62312 23 176314 25 1424 1 558 1010 5 1424;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 2 2 1;0 29 27 0 33 33 0 21 22 33 0 0 0 33 0 0 29 27 0 0 0 1 0 1 2 2 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;19933 15729 21796 2512 2 315 2 35 19 2 41 72;0 0 21 0 0 0 0 0 0 0 1 2;0 0 27 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;14 5311 618 16 5311 618 446 1 1 4 493;0 0 0 0 0 0 0 0 0 1 2;0 13 14 0 13 14 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;4330 1912 2 4330 344 1912 2 40993 1 72271 1 41444 14726 1 41444 33384 1 18149 33384 1 75364;9 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 33 0 13 1 2 0 21 0 0 0 21 22 0 0 55 0 33 55 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11 1496 110 366 101 63 380 110 6053 3 10128 4485 101 17 1 4 9;0 9 0 0 0 0 9 10 10 0 0 0 0 0 0 1 2;0 13 33 0 0 0 13 14 14 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4369 1577 301 8 190 488 41 10349 8 302 806 8 629 492 11 82 9830 3368 8 9830 3368 8 1402 3143 8 1402 3143 8 9830 134 10 1163 3 8 404 375 83 1 4 9;0 27 28 0 0 0 0 0 0 0 0 0 0 0 0 0 33 34 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 33 34 0 0 0 0 33 0 33 0 0 0 1 2 0 1 2 0 33 34 0 33 34 0 0 0 0 0 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 7046 22187 48 53 53 6477 33103 57 0 2 0;19 20 20 20 0 0 0 0 0 0 0 0;0 21 22 0 33 34 33 33 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1; -;3682 101 1726 1050 93 363 0 2 0 858 197 334 100;0 0 0 0 0 0 0 0 0 17 0 0 0;11 12 0 33 0 0 0 0 0 31 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1671 434 1 650 94 18907 8 650 467 94 730 8 650 467 18907 8 650 467 800 99 8 650 94 730 8 650 18907 8 650 94 183 8 650 94 730 104 8 650;0 0 0 9 0 0 0 9 0 0 0 0 9 0 0 0 9 0 0 0 0 9 0 0 0 9 0 0 9 0 0 0 9 0 0 0 0 9;33 33 0 13 33 34 0 13 21 22 33 0 13 33 34 0 13 33 0 0 0 13 33 34 0 13 33 0 13 33 34 0 13 33 34 33 0 13;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8893 22467 192 119 286 62 92 66 28 18 1 14 443 71 1 1489 71 1 908 71 1 434 71 1 123 147 1 116 71 16 196 2 1841 510;21 22 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 33 34 33 0 13 0 33 34 0 0 33 34 0 33 33 0 33 34 0 33 33 0 33 33 0 33 34 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;943 158 146 7 228 81 17512 200 2205 2 379102 251 3826 1529 88383 1 97065 276 2 78 2 35 979 19 566 41 72;45 46 0 0 0 0 0 0 0 0 21 22 22 22 22 0 0 0 0 0 0 0 0 0 0 1 2;53 54 0 0 0 0 33 34 34 0 27 0 33 34 34 0 0 0 0 33 0 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1835 948 17891 16 1 1835 948 17891 87 1 1835 948 17891 104 269 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;0 33 33 33 0 0 33 33 33 31 0 33 33 33 33 34 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1038 9591 257 4927 306 5388 37157 967 2588 449 13 967 238 296 0 2 2 0 1962 5 0 1962 69 5;17 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 1 2 2;31 32 32 33 0 33 33 0 0 0 0 0 0 33 0 0 0 0 1 2 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7415 3 81 2055 945 245 1 1 44 926 1 677 1 945 245 1930 1 3079 68 5 3171 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0;35 36 36 36 33 34 0 0 33 33 0 0 0 33 34 33 0 1 2 2 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;360 38 2058 6905 621 1388 0 360 38 2058 6905 3072 621 1 629 2816 0 2 0 2956 358;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 22 33 33 35 0 21 22 22 33 33 33 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7489 9567 106585 0 11799 148 16542 42 16161 3461 13878 2 11799 1456 5;0 0 0 0 9 10 0 0 0 0 0 0 0 1 2;21 22 22 0 13 14 33 0 33 0 33 0 33 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1015 4004 285 1991 206 3 285 201;0 0 0 0 0 0 0 0;33 33 33 33 0 0 35 36;1 1 1 1 1 1 1 1; -;3817 5909 1016 1 4 61;0 0 0 0 1 2;33 34 33 0 1 2;1 1 1 1 1 1; -;139 655 144 2756 23154 177 21823 844 26 6393 10719 1860 1081 162 264 0 2 0 2756 0 17738 0 2 0 81 939 100;0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 17 0 0 0 0 0 0;0 33 0 29 30 0 31 33 0 0 33 21 22 0 0 0 0 0 31 0 31 0 0 0 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1942561 687 633 2 1942561 1032 19 2 41 72;0 0 0 0 0 0 0 0 1 2;0 33 34 0 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1; -;102 38 1103 92 27 19126 1456 4891 1678 1188 7 2210 7 2210 1 26071 5;0 0 0 9 10 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 13 14 21 22 22 33 33 21 22 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;912 1342 1942 345 44794 6 912 1342 181 1403 2181 3330 2181 6 2181 799 90 345 199 912 1342;0 0 19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;9 10 25 33 31 0 9 10 33 34 0 33 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;233 4587 8139 37 1 137 233 4587 8139 37 26 104 307 1 233 4587 8139 91 180 0 2 0 318 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;29 30 33 33 0 21 29 30 33 33 0 33 33 0 29 30 33 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;744 1395 13501 610 1951 917 435 138 176 4091 0 2 435 4091;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 0 33 33 0 33 34 0 33 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;163360 207 928 18 0 2 277;17 18 18 18 0 0 0;0 31 32 32 0 0 0;1 1 1 1 1 1 1; -;1874 1329 2140 21018 10044 6538 731 2 2140 21018 10044 3 6538 731 1 242 852 5;9 9 10 10 10 0 0 0 9 10 10 0 0 0 0 1 2 2;13 13 14 14 14 33 34 0 13 14 14 0 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7959 146 417 1661 3035 13333 536890 1 405 1 35 19 31 95;0 0 0 21 22 0 0 0 0 0 0 0 0 0;53 0 0 27 28 33 27 0 1 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34 9805 337 1802 1 1768 1 188 5652;17 18 18 18 0 0 0 17 0;31 32 32 32 0 33 0 31 33;1 1 1 1 1 1 1 1 1; -;510771 5811 76 1949 176 2621 4215 19 2 458 2 35 19 2 41 72;23 24 24 0 0 0 0 0 0 0 0 0 0 0 1 2;29 30 30 33 0 33 33 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;277 2 248 2055 2248 260 178 2 322 5;0 0 0 0 0 0 0 0 1 2;0 0 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;11 52 29 97 89 212 1254 21 6 1645 8216 11 145 6750 79270 6 6644 11 11860 6 10 15 441 1 4 79 9;0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0;0 33 34 33 34 0 33 0 0 33 33 33 34 0 0 0 33 0 33 0 0 0 33 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1203 10 80 2743 3327 32 425 5551 1043 0 1520 60 60 1 4 9;0 0 0 0 0 0 0 31 32 0 0 0 0 0 1 2;33 33 34 33 34 0 0 39 40 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;545 1129 108 707 101 2131 1 1129 108 707 5835 0 2 0 545 707 101 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 33 34 33 34 33 0 33 34 33 34 0 0 0 21 22 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;31243 0 22495 17088 0 86851 0 231222 0 6282 64579 0 7224 0 616002 3555 0 59407 0 107341 0 89926 0 11193 0 105743;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;27 0 21 33 0 0 0 0 0 0 33 0 33 0 33 0 0 21 0 33 0 0 0 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;139 2865 85 655 1760 144 47340 1216 7960 8 73 883 32 1152 1599 1 4 815;0 45 46 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 53 54 33 33 0 33 0 33 0 0 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1648 4700 335 446 16 1 1648 4700 335 844 327 739 281 2073 1 4 493;0 17 18 18 0 0 0 17 18 18 0 0 0 0 0 0 1 2;0 31 32 32 33 0 0 31 32 32 33 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3439 838 0 3439 838 40 0 170 3439 838 40 2 8885 7 61137 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 0 21 22 33 0 33 21 22 33 0 53 0 1 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1942561 6 0 37473;0 0 0 0;0 0 0 33;1 1 1 1; -;10645 16030 81 155 329 0 440 154 421 16030 9446 29307 0;29 30 0 0 0 0 0 0 0 29 0 0 0;37 38 0 33 34 0 33 0 0 37 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;88 4480 60729 119 28 18 270 383 1972 242 7 679 1196 869 170 367 74 31 136;17 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0;31 32 32 32 32 32 33 34 33 34 0 21 22 33 33 33 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;459 298 106 29 97 450 3847 23123 3 746 509 862 24 17 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;33 0 35 36 36 0 0 33 0 33 0 33 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3479 1229 86 4355 989 0 19261 15 321 1 4 9;5 6 6 0 0 0 0 0 0 0 1 2;9 10 10 33 21 0 55 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;2074 33416 1613 1 2074 33416 393 3799 1497 189 14 4121 12867 5 16;23 24 0 0 0 0 0 0 0 0 0 1 2 2 0;29 30 21 0 29 30 0 33 34 34 0 1 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;913 106 376 82 49908 83 1255 7787 0 45731 5132 82 8911 604 83 2 19447 27 169 287 5;9 10 10 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 0;13 14 14 0 0 0 0 0 0 0 33 0 33 0 0 0 13 14 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;511 74 162 1568 2497 3 7921 1835 1 296422 1 114 204;0 0 0 0 0 0 11 12 0 0 0 1 2;0 0 0 0 33 34 15 16 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2521 2108 2214 0 87 8371 104 0 1625 20867 12031 8371 0 8371 104 0 10094 16 37 6 87 6 44 6 8371 26 5290 74 61225 6 2521 2108 2214 10591 28 18 2 395 5;0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 0 9 17 18 18 18 18 0 1 2;0 13 0 0 0 31 33 33 0 33 33 33 33 0 33 33 0 11 0 33 0 31 0 33 0 33 0 33 0 29 0 13 0 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4093 11880 2011 46 198 38 298 5388 16954 353 15567 9918;0 0 0 0 0 0 0 0 0 0 0 0;33 33 33 0 33 34 0 33 33 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1; -;14 467 1230 6094 117 563 4756 16 37 6 87 6 44 6 467 1230 6 157 27 181 179 3224 192 28 18 2 395 5;0 23 24 0 0 0 0 0 0 0 0 0 0 0 23 24 0 17 18 18 18 18 18 18 18 0 1 2;0 29 30 33 0 0 33 0 33 0 31 0 33 0 29 30 0 13 14 33 34 33 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;913 164 376 1 709 253 0;9 10 10 0 0 0 0;13 14 14 0 1 2 0;1 1 1 1 1 1 1; -;22 1992 283 386 122 21 8 1290 13 820 3246 135 16370 8 10 15 441 203 17 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;33 34 33 34 21 22 0 33 33 34 33 0 33 0 0 0 33 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;447 50 1239 805 28 18 6056 246 18 0 2 0 2251 1931 172 1270;17 18 18 18 18 18 18 18 18 0 0 0 21 22 0 0;31 32 32 32 32 32 13 33 34 0 0 0 27 28 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;236 227 322 5 2028 8 1257 9543 26 490 3875 21 8 236 227 322 5 2028 14692 1257 9543 26 490 1887 322 5 2028 2831 551 84 222 440;0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 1 2 0 23 0 0 0 0 0 1 2 0 23 24 0 0 0;0 0 1 2 33 0 0 33 0 0 33 0 0 0 0 1 2 33 33 0 33 0 0 0 1 2 33 29 30 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1119 7129 3674 1 4 253;17 18 18 0 1 2;31 32 32 0 1 2;1 1 1 1 1 1; -;22 134 89 15 155 63 597 597 963 801 7649 1132 21 597 597 1378 165 350 16313 22 11750 36171 3 597 597 1476 22 12 70 155 4851 597 597 2814 22 136 0 2 0 4118 253 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 0 0 0 0 0 5 6 0;0 0 0 33 34 0 33 34 33 33 33 33 34 33 34 33 0 0 33 34 33 33 0 33 34 33 34 33 34 0 33 33 34 0 33 34 0 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2689 76 30811 86 2982 1979 653 145 16 37 6 87 6 44 6 30811 267 74 30811 86 6 157 27 6406 1870 11654 4685 81 2378 192 1172 244 2 395 5;0 23 24 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 10 9 10 0 17 18 18 18 18 18 0 1 2;0 29 30 33 0 13 33 33 34 0 33 0 31 0 33 0 33 33 0 33 0 0 13 14 13 14 0 0 33 34 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5973 5448 6599 51 1879 2056 5448 0 23 1808 25 0;0 0 0 0 19 20 20 0 0 1 0 0;25 26 26 0 33 34 34 0 0 1 0 0;1 1 1 1 1 1 1 1 1 1 1 1; -;742 59 59 59 132 165 22 6202 341 21 2604 194 2283 59 402 659 1822 902 33 24 17 1 4 9;0 0 0 0 0 0 0 29 0 0 37 38 38 0 0 0 0 0 0 0 0 0 1 2;0 0 0 0 33 33 34 33 33 34 45 46 46 0 33 0 0 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;220 386 496 2176 12 527 1961 1908 8403 348 58 13 4574 13 4574 19241 32 7277 12 3876 23700 1 12 1262 1 33 505 31 2 624 463 5 107 344 372;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0;0 0 33 33 34 34 33 33 33 0 0 0 0 0 0 33 0 33 0 33 33 0 33 34 0 1 2 2 0 33 34 34 33 21 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;140 2299 1725 1 8655 2647 46 10171 5701 81 10526 81 544 1 3647 140;0 0 0 0 0 0 0 0 0 0 0 17 18 0 1 2;33 34 0 0 33 33 0 33 33 0 33 31 32 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2754 1581 55 548 7 228 46 15067 2394 282 1 2754 1581 55 548 7 228 46 15067 2394 300 55 1581 78 40;45 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;53 54 54 33 0 0 0 35 36 33 0 53 54 54 33 0 0 0 35 36 33 34 33 34 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9480 1675 7005 998 9068 10 10010 203 7 7 7 0 2 0 49 2658 3013 117 46 7206 48 0 2 0 4761 4886 100 0 2 0 956 0 556 0 1157 232;0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 46 46 46 46 46 46 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0;33 33 34 0 33 0 33 0 0 0 0 0 0 0 53 54 54 54 54 54 54 0 0 0 1 2 33 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 9 126 61575 10 654 1202 2595 3 6204 17 1 4 9;0 0 0 0 0 31 32 32 0 0 0 0 1 2;33 34 0 37 0 39 40 40 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;32462 2 100479 180833 40410 10 20 156 1 32462 2 100479 180833 40410 3 273 1 3019 1 1809 1 1526 1 2139 1 41 5936 31 1538;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;31 0 0 33 33 33 34 33 0 31 0 0 33 33 0 33 0 21 0 33 0 33 0 33 0 9 10 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;933 567 1590 0 6541 2017 22 212 21 0 3 845 202 1 4 9;9 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 33 33 0 33 34 33 34 34 0 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4837 4267 46 757 1070 17797 508 13818 3195 2873 1 8748 297 1 114 268 1 114 5;17 18 0 0 0 0 0 0 0 0 0 23 24 0 1 2 0 1 2;31 32 0 33 34 33 33 33 33 33 0 29 30 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1942561 0 123 4009 196 36 94 730 3630 0 2 0 6623 43;0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 34 33 0 33 34 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1252 17394 3 613 1 4 209 132;0 37 0 0 0 17 23 24;0 45 0 33 0 31 29 30;1 1 1 1 1 1 1 1; -;2377 5794 53 53 163296 1 1177 142 5;0 37 0 0 13 0 1 2 2;33 33 33 34 17 0 1 2 2;1 1 1 1 1 1 1 1 1; -;3229 233 404 3 11 1361 3458 24 17 0 2 0 612 295 42;0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;21 22 33 34 0 0 33 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 2885 3 634 62 936 324 66 23 534 3897 724 25 48 2038 46 7447 373 301 3828 7 5764 1 1115 40;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 34 34 0 33 34 0 0 33 33 33 0 0 33 0 27 28 33 33 0 21 0 9 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;157 2102 5608 6 157 13710 5608 6 157 4573 5608 53 157 1142 2102 132423 210 565 1392 1951;9 23 24 0 9 0 23 0 9 23 24 0 9 0 0 0 0 0 0 0;13 29 30 0 13 33 29 0 13 29 30 0 13 33 33 33 33 34 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3251 7029 3 1107 5153 625 15 2281 222 9093 1021 1 3251 7029 1 7019 100;45 46 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0;33 34 0 33 33 0 0 33 0 33 0 0 33 34 0 1 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3053 41 7443 5958 53 53 88 1362 98 3174 718 30561 1 718 30561 1 34 1853 5055;0 0 0 0 0 0 17 18 18 18 18 0 0 0 0 0 9 17 0;35 36 0 0 33 34 31 32 32 32 32 33 0 33 33 0 13 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;174 936 79 238 29 97 10 3875 93 9846 24 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 21 33 34 34 0 33 0 33 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 226437 190965 431 3793 57 16 226437 62 1311787 66 0 190965 431 3793 57 0 2493 26 728 26 15021 26 948 260 14084 76 26 782 2055 76 26 160 2859 76 14 1236 0 307 0 37 0 1396 16 2 553;0 23 24 24 24 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 0 23 24 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;0 29 30 30 30 30 0 0 0 0 0 0 0 33 13 0 0 33 0 33 0 33 0 21 22 29 30 0 29 30 30 0 0 0 0 0 33 0 33 0 33 0 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 963 554 2545 3326 6657 0 1494 506 2487 24 781 1 2375 499 382;0 0 0 37 0 0 0 0 0 0 0 0 0 0 0 0;0 33 33 34 33 33 0 33 33 33 0 33 0 21 22 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 3522 17302 46 308 1158 213 85 48 12040 2237 76 3781 46 529 22774 11539 45788 1 3333 5 0 9781 8059 7 1054;19 20 20 20 20 20 20 20 20 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;0 33 34 0 33 34 34 0 0 33 0 0 33 0 0 33 33 33 0 1 2 0 27 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4 280 53 53 317 1264 607 1277;1 2 0 0 0 0 0 0;1 2 33 34 0 33 0 33;1 1 1 1 1 1 1 1; -;33 505 31 1 34 77 50 3 1023 128;1 2 2 0 9 0 0 0 0 0;1 2 2 0 13 33 34 0 33 33;1 1 1 1 1 1 1 1 1 1; -;370 94 56 11 29 94 78 13 888 10 29 4365 365 32 22826 418 93 800 8 216 2985 58 1500 1460 1500 1675 886 4686 4763 8 2815 180 122 21 373 333 1 4 9;23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;29 30 0 0 0 33 34 33 34 0 0 33 34 0 21 22 0 0 0 21 22 0 0 33 0 0 0 33 33 0 33 33 34 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1179 27 102 38 1266 745 1029 62 293 361 66;9 10 0 0 0 0 0 0 0 0 0;13 14 33 34 33 34 33 0 0 33 0;1 1 1 1 1 1 1 1 1 1 1; -;14 157 8797 10604 983 37 1 10604 983 44 1 10604 983 1303 71 16 2 157 631 5;0 9 9 29 30 0 0 29 30 0 0 29 30 0 0 0 0 1 2 2;0 13 14 37 38 33 0 37 38 33 0 37 38 33 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;273554 6522 58 47217 2298 275 4333 1655 2627 2 34 2656 7334 414 18;21 0 0 9 10 10 0 0 0 0 17 18 18 18 18;27 33 34 13 14 14 33 0 0 0 31 32 32 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4065 58 607 33 214 574 492 171 619 240 21 8 847 165 317 309 3211 8897 63 106 246 122 8 135 2815 455 839 1 4 9;33 34 34 34 33 34 34 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;55 56 56 56 33 34 34 33 0 0 0 0 33 34 0 0 33 0 0 33 34 0 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;105 2 7827 38 34 13322 2840 552 3047 294 67 383 2706 2085 343 1 4 61;0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 34 13 33 33 33 33 33 0 33 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1112 237 833 4484 242 0 2 0 364 5;0 0 0 0 0 0 0 0 1 2;0 33 33 34 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;404 3718 351 46 119 317 1517 607 5038 0 1241 120 13611 323 540 289 2560 1 841 69 1 1368 5;1 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;1 2 2 0 33 0 33 0 33 0 33 34 31 0 0 0 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1400 1593 912 1896 54068 0 2 0 2628 649 141 2 315 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 21 33 33 0 0 0 33 33 34 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1639 819 18 2272 9175 272 417 82 2517 1503 687 83;0 0 0 0 0 0 0 0 0 0 0 0;33 34 33 33 0 33 0 0 21 22 22 0;1 1 1 1 1 1 1 1 1 1 1 1; -;126 22 1490 21 5036 327 2783 3 41 64 337 64 12 21 41 64 1302 358 5036 327 2841 227 322 60 0 2 0 612 295 42;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 0 0 0 1 2 2;21 22 33 0 33 0 33 0 0 0 33 0 33 34 34 0 33 0 33 0 0 0 1 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;64 1173 70 2327 86 1782 530 8796 3 233 284 1184 7682 3768 63 4193 7682 6 2065 233 5839 21 91 7969 60 233 3 28070 2281 21 91 60 2255 10 39 127 217 1 4 130 112;0 23 24 24 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 29 30 30 30 0 33 33 0 0 0 33 0 33 0 33 0 0 0 0 33 0 0 33 0 0 0 33 33 0 0 0 33 0 33 34 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16753 1870 7009 241 1019 21986 258 0 2 0 324 308 0 2 0 1637 1714 5 6 680 1246 491;9 10 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0;13 14 33 33 34 33 0 0 0 0 33 34 0 0 0 1 2 2 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;20578 19925 1 6950 20578 19925 1900 280 6 3792 1 20578 19925 30 215 6 2054 24 2 3792 499 1900 280 5;9 10 0 9 9 10 0 0 0 0 0 9 10 0 0 0 0 0 0 1 2 0 1 2;13 14 0 13 13 14 33 34 0 33 0 13 14 33 34 0 33 0 0 1 2 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;23914 22 3 1847 783 1 4 61;0 0 0 0 0 0 1 2;33 33 34 34 0 0 1 2;1 1 1 1 1 1 1 1; -;73 484 406 44 6 35 6396 1 2349 24686 926;0 0 0 0 0 0 0 0 0 0 0;35 36 36 33 0 33 0 0 33 33 33;1 1 1 1 1 1 1 1 1 1 1; -;650 17364 8691 2204 565 63 2425 1294 199 1 4 9;17 18 18 18 18 0 9 10 10 0 1 2;13 13 33 33 34 0 13 14 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;1860 12001 47754 1390 3 1156 3213 0 2 0 738 4906 738 609;0 0 0 0 0 0 0 0 0 0 17 18 0 0;0 0 33 34 0 33 33 0 0 0 31 32 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;139 3463 144 800 4954 2965 14431 4156 32 82 173 1091 2824 33868 83 3 13914 3322 1 42453 1 114 204;0 0 0 0 0 0 0 0 0 0 0 0 21 22 0 0 0 0 0 21 0 1 2;0 33 0 0 0 33 33 33 0 0 0 0 0 0 0 0 33 33 0 27 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16961 15 216 22211 203 1 316 3005 1034 2864 16961 43 1 4 47;21 0 0 0 0 0 45 46 46 46 0 0 0 1 2;27 0 0 33 0 0 33 34 34 34 27 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 28292 57716 2225 390 69 1 28292 57716 2225 390 4317 1 110 54 115;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18;0 0 0 11 12 21 0 33 0 11 12 21 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9337 1634 54 19557 16153 178 2130 1146 1 5565 4747 9594 19 1 5502 315;45 46 0 0 11 0 0 0 0 0 0 0 0 0 0 0;53 54 0 33 33 0 33 33 0 53 54 54 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;52 2311 567 32 1574 37 7654 6351 228 7 57 764 2 247 268;0 0 0 0 0 0 0 0 0 0 0 0 0 17 0;0 33 33 0 33 33 21 22 0 0 33 34 0 31 21;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 39735 60541 0 4136 6371 16 307 1 927 1 44 1 100 1 23 227119 25 39735 60541 0 4136 6371 4086 184 307 2 2566 1533 31;0 23 24 24 24 24 0 0 0 0 0 0 0 0 0 0 17 0 23 24 24 24 24 23 24 0 0 0 1 2;0 29 30 30 30 30 0 33 0 33 0 33 0 33 0 0 31 0 29 30 30 30 30 29 30 33 0 1 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1267 3361 6212 2374 2 4972 191 573;9 10 10 0 0 9 1 2;13 14 14 33 0 13 1 2;1 1 1 1 1 1 1 1; -;2114 1334 603 5871 15 273 203 17 0 2 0 296 5860 0 2 0 4904 0 2 0 2240 0 2696 0 709 197;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 1 2;0 0 0 33 0 33 0 0 0 0 0 33 33 0 0 0 31 0 0 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 338613 77657 2133 223 183 592 76 16 37 1 87 1 44 0 2 427 262 318 5;0 0 0 0 23 24 24 24 0 0 0 0 0 0 0 0 0 0 1 2;0 33 29 33 29 30 30 30 0 33 0 31 0 33 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 29 1267 1358 317 9191 4508 17965 393 3 4259 1 1441 2 806 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 33 0 33 0 33 0 0 33 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2157 99 32 371 583 89355 3 454 59 742 59 1 4 9;23 24 0 0 0 0 0 0 0 0 0 0 1 2;29 30 0 21 22 22 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2102 11615 30 225 40 1 4 9;0 0 0 0 0 0 1 2;33 33 33 34 33 0 1 2;1 1 1 1 1 1 1 1; -;7976 18 906 134 11 256 384 60 1 4 9 0;0 0 0 0 0 0 0 0 0 1 2 0;33 33 33 0 0 0 33 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1; -;49 1307 3787 1820 8351 3 530 29 5038 406 266 52 62 5914 406 266 260 941 8 3191 15298 117 38 8 3285 151 353 1307 174 284 45 11175 8 10794 898 2859 59 186 266 17 90 96 3 214;0 27 28 28 28 28 28 28 28 28 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 35 36 36 36 36 36 36 36 36 36 0 0 33 34 34 33 33 0 33 33 33 34 0 33 33 0 33 34 33 34 33 0 33 34 34 0 0 33 0 55 56 56 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5613 115 2762 11 119 242 6657 324 6 560 242 3 324 549 514 119 242 122 283 246 54 127 7 5613 115 2762 3 560 242 514 119 242 122 91 324 60 23 1315 309 3335 698 4631 887 6 555 3212 25 1 4 130 112;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;27 33 34 0 33 0 33 0 0 33 34 33 34 0 0 33 0 0 33 34 34 34 0 27 33 34 0 33 34 0 33 0 0 0 0 0 0 0 0 33 34 33 33 0 0 33 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4392 18540 1523 4392 28215 4801 974 10 27062 7 7765 10 20 156 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;0 33 34 34 34 34 0 0 0 0 33 33 34 33 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3811 82 1475 577 2409 83 1245 45 3 81 229 176 4998 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 34 34 0 33 33 34 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 23474 299 849 101 516 269 36 23474 299 849 101 2003 16 0 2 0 2158 702 1579;0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 9 17 18;0 13 33 11 12 33 34 0 13 33 11 12 33 0 0 0 0 13 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 744 71 5109 48 367 361;0 0 0 0 0 0 0;0 33 34 35 0 33 33;1 1 1 1 1 1 1; -;3199 3 16246 15 20553 26 80 5192 3199 3 16246 26 250 5192 3199 3 16246 1 1469 400 3955 1 1780 54 115 5;0 0 37 0 0 0 0 0 0 0 37 0 0 0 0 0 37 0 0 0 0 0 1 2 2 2;0 0 45 0 33 0 0 33 0 0 45 0 0 33 0 0 45 0 33 34 33 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11948 1822 4843 3209 795 30 906 10 12 10 135 1934 3209 121 3910 1 4 9;5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;9 0 33 33 33 0 33 33 34 34 0 33 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;90 3157 12 3157 614 509 730 216 123 0 17958 38253 1 31337 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 0 0 33 33 34 0 33 0 0 13 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;340 102 38 118 58 5388 256 1161 394 150 3182 17 2 2 340 27 394 5488 69 696 153;9 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 18;13 33 34 33 34 33 0 33 34 34 33 0 0 0 13 14 31 32 32 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1226 372 230 10 80 14770 3 17 0 2 0 1268 3533 3 1487 0 2 0 605;0 0 0 0 0 0 0 0 0 0 0 21 22 0 0 0 0 0 1;33 34 0 33 34 0 0 0 0 0 0 33 34 0 33 0 0 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12547 3885 992 3 11 5171 24 17 1 12547 358 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 33 34 0 33 0 0 0 37 38 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;142 310 108 458 1025 1 761 5;0 0 0 0 0 0 1 2;33 34 33 33 34 0 1 2;1 1 1 1 1 1 1 1; -;14 2442 2691 16681 446 16 1 2442 2691 2163 5680 74 6960 778 1 4 493;0 0 9 10 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 33 33 0 0 33 34 33 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;768 2283 21 15 1082 6971 5189 249 79 1 4 79 9;0 37 0 0 0 0 0 0 0 0 0 0 0;33 33 0 0 33 33 33 0 21 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1; -;11145 22517 0 14 19466 16 0 2 0 188 271 202;19 20 0 0 19 0 0 0 0 5 6 6;33 34 0 0 25 0 0 0 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1; -;402 133 228600 133 817 20 133 228600 133 206 3 1809 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 0 0 0 0 0 0 0 0 0 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34 169 1439 8 5768 134 17425 8 526 17425 3 180 160 215 21 1 297 186 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 0 0 0 33 0 0 33 0 0 0 0 0 0 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;95124 481;0 0;0 33;1 1; -;58846 81 354 45 1402 1247 165 41 0 12 6471 2138 19841 23 68 25 1 167 131 1 34 5;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;27 33 34 0 33 34 34 34 0 0 33 33 33 0 0 0 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;760 8684 3 5793 77770 4026 1 2087 5;0 0 0 0 0 0 0 1 2;0 33 0 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1; -;58069 10612 714 53399 13 4604 23 68 25 1 188 69 108;0 37 0 0 0 0 0 0 0 0 17 18 18;33 45 0 0 33 34 0 0 0 0 9 10 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 20081 0 72076 2 66263 0 14843 2 22880 5 897 100 0;0 17 0 0 0 0 0 0 0 1 2 0 0 0;0 29 30 30 0 0 0 29 0 1 2 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 239 30566 193 40 16 137 9560 0 239 30566 193 170 40;0 0 0 0 0 0 0 17 0 0 0 0 0 0;0 0 33 21 22 0 21 31 0 0 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1483 6453 311 14 7635 16 0 6453 1607 3 7581 17247 59 59 59 59 1 3444 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 35 36 0 0 0 0 33 34 0 33 33 0 0 0 0 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;780 90 23382 1 120378 276 2 841 45547 3190 1118 22852 0 2 1031 2 35 979 19 2 41 72;19 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;25 26 26 0 0 0 0 33 33 33 33 33 0 0 33 0 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 2108 18111 2957 20171 16 1 2108 18111 2957 20171 37 1 2108 18111 2957 20171 104 1 2108 18111 2957 20171 87 1 395 5 0;0 0 31 0 31 0 0 0 31 0 31 0 0 0 31 0 31 0 0 0 31 0 31 0 0 1 2 0;0 0 39 0 39 0 0 0 39 0 39 33 0 0 39 0 39 33 0 0 39 0 39 31 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;550 38 24258 18207 108 7020 3484 6949 53 31 291 53 387 5 8 19 35 31 95;0 0 17 18 18 0 0 0 0 0 0 0 1 2 0 0 0 0 0;33 34 31 32 32 33 34 33 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14626 43 46 20 876 93 1382 20378 17 1 39 13 268;0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 0 0 33 0 33 33 0 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1; -;65835 115318 122948 1690 57 36 378 590 63878 65835 115318 122948 1690 57 1825 849 460 4842 36 398 36 307 36 44 0 849 101 0 34 110 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 7 8 0 0 0 0 0 0 0 0 0 0 1 2 2;21 0 21 33 34 0 33 0 33 21 0 21 33 34 0 0 11 12 0 33 0 33 0 33 0 11 12 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4558 1221 1 4558 1221 37 1 410 4558 1221 104 26 182 0 2 0 111;11 0 0 11 0 0 0 0 11 0 0 0 0 0 0 0 17;15 0 0 15 0 33 0 33 15 0 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14070 514 22679 1 4 253;0 0 0 0 1 2;21 22 22 0 1 2;1 1 1 1 1 1; -;6088 1228 152 1217 6 799 12 76167 1 4 9;0 0 0 0 0 0 0 0 0 1 2;0 33 0 0 0 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;59909 46 22 236 2272 34 9475 3 23837 42 54 13 1 188 119;21 0 0 0 0 9 9 0 0 0 0 0 0 17 18;27 0 33 34 33 13 13 0 33 34 0 0 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10483 18 14166 16803 758 305 434 2526 12029 0 2 0 738 4906 738 372;0 0 0 0 0 0 0 0 0 0 0 0 17 18 0 0;33 34 33 33 33 33 33 34 33 0 0 0 31 32 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;17761 675 297 1 709 253 0;0 0 0 0 0 0 0;21 22 22 0 1 2 0;1 1 1 1 1 1 1; -;14 2482 1287 3227 16 1 2482 1287 3227 37 1 2482 1287 3227 104 1 2482 1287 3227 87 1 395 5 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;0 33 34 34 0 0 33 34 34 33 0 33 34 34 33 0 33 34 34 31 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;79101 1211 2629 13 522 6 146 173 6 151 173 6 431 173 6 1145 13 522 1211 2629 196 1 4675 1211 2629;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 25 26 33 34 0 33 34 0 33 0 0 33 34 0 33 9 10 10 10 33 0 0 25 26;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9278 0 41908 0 2602 266 14311 0 2 0 19473 3 12438 0 2 0 204 292;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 0 33 33 33 0 0 0 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;102 984 750 168 7155 1388 3464 46 121 15847 68872 145 3309 1 4 61;0 9 39 40 0 0 0 0 0 0 0 0 0 0 1 2;33 13 47 48 47 35 33 0 0 21 22 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;83658 50 290 15 124 1 83658 50 290 416 865 408 5724 57 2364 8 2 2659 5;17 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 0 0 0 0 33 34 33 34 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2424 88 27 4243 75 2655 521 510 7105 2496 4775 1029 1 4 61;0 9 10 9 10 0 0 0 0 0 0 0 0 1 2;33 13 14 13 14 33 34 33 34 34 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;21592 8060 472 29 97 118 762 900 42 5482 46 1212792 1948 7294 154 85 1891 2 247 69;0 0 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 0;33 34 33 0 27 28 0 33 34 0 0 27 0 0 33 34 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;511 241 1126 1726 1 4 9;0 0 0 0 0 1 2;0 0 0 0 0 1 2;1 1 1 1 1 1 1; -;80 1133 1027 3 63 50 1268 3 4806 60 3560 30 16067 109 1027 93 3063 50 1268 3 4806 7 0 2 0 612 295 42;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 0 33 0 33 34 0 0 33 0 33 0 33 0 33 0 33 33 34 0 33 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;445 437 49597 2 210 2 35 19 2 41 72;39 40 0 0 0 0 0 0 0 1 2;47 48 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;8680 1995 8167 1 31 1688 1 31 1688 5 1 632 400 1688 1 9338 1688 1 2932 1242 5;31 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;39 40 33 0 33 34 0 33 34 0 0 21 22 22 0 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11 13 29 1597 8 1001 13989 190 488 8 999 190 488 1001 89 11 166 350 3007 4659 8 8269 1868 190 488 762 9 350 3 3570 8 350 3 13989 59 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 33 0 33 34 33 34 0 0 33 34 33 33 34 0 0 33 33 0 33 34 33 34 21 22 33 34 33 0 33 34 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22852 4763 2627 527 58 12 58 166 1908 12 33 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 0 0 33 34 34 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15 3356 2680 4352 762 12 1648 17 129 422 2722 317 1228 17022 72310 7851 59 2 2680 36 4352 2 1905 5 207;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;0 0 21 22 0 0 0 0 0 0 33 0 33 33 34 34 0 0 33 0 33 0 1 2 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 10632 569 48 755 33433 725 68573 8 2712 659 13 10782 7764 8876 3 12613 37299 59 1 69 1 12044 3309;45 46 46 46 0 0 21 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 34 0 33 27 28 0 33 0 33 34 33 34 0 33 33 0 0 33 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1410 6558 64 82 7637 14550 83 369 2463 12917 0 106 353 122 42 1574 2241 9308 1 188 69 108;9 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;13 33 0 0 29 30 0 0 33 0 0 0 33 34 0 33 0 33 0 9 10 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1269 345 808 1546 1 4 61;0 0 0 0 0 1 2;33 34 33 34 0 1 2;1 1 1 1 1 1 1; -;5687 5 2 1389;0 0 0 0;0 0 0 33;1 1 1 1; -;105 38 57 97 3254 663 3266 24289 1018 2732 832 59 0;0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 33 34 0 33 0 33 0 33 33 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;26548 837 81 5382 13583 3147 3259 1891 0 3625 3441 1791 20187 1 188 268;17 18 0 0 0 0 0 0 0 0 0 0 0 0 23 24;31 32 33 34 33 34 0 33 0 33 34 33 33 0 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;139 389 69 5283 144 113 3172 1056 632 830 688 394 583 2850 4529 2280 26054 1 69 131 1 1337 5 23 2867 7 65 25;0 19 20 20 0 0 0 0 0 0 0 0 0 0 0 9 9 0 0 0 0 1 2 0 0 0 0 0;0 25 26 26 0 0 33 34 33 34 0 33 0 33 33 13 13 0 31 32 0 1 2 0 31 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1796 53685 1469 381 5425 0 1796 0 1469 381 0 823 33 0 1 1113 1186 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;13 33 33 33 33 0 13 0 33 33 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;34 461 218 27 4008 75 4008 75 3814 6992 873 23505 56 503 25387 1267 99 642 5437 8 218 4462 1306 9550 28 18 1461 108 3 147 0 2 0 218 469;9 0 9 10 9 10 9 10 9 10 10 10 10 10 0 0 0 0 0 0 17 18 18 18 18 18 18 18 0 0 0 0 0 27 28;13 13 14 14 13 14 13 14 0 0 0 33 33 34 0 33 0 0 0 0 13 0 0 33 33 34 33 34 0 33 0 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;84169 14217 3726 161 508 0 2 0 421 499 480 480;21 0 0 0 0 0 0 0 1 2 2 2;27 33 33 33 34 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1; -;1251 56 77 1116 34011 0 137 1327 0 195 2829 458 0 2 0 7748 301;27 28 28 28 28 0 0 0 0 0 0 0 0 0 0 1 2;35 36 36 36 36 0 21 33 0 0 1 2 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;598 27 8823 1878 565 0;17 18 18 18 18 0;31 32 32 32 32 0;1 1 1 1 1 1; -;34 3446 42742 1244 7707 278 12745 46 338 4260 1287 3 1730 20238 59 1 25814 5;9 0 0 0 0 0 0 0 9 0 0 0 0 37 0 0 1 2;13 33 33 33 33 34 34 0 13 33 0 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;23736 67 31700 0 2 0 364 5;0 0 0 0 0 0 1 2;25 26 26 0 0 0 1 2;1 1 1 1 1 1 1 1; -;250 5828 385 10 5680 194 274 859 194 9443 17 1 4 209 132;0 0 0 0 0 0 0 37 38 38 0 0 17 23 24;0 33 0 0 0 0 0 45 46 46 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4330 63 11616 1128 110 15 589 1 4 9;9 0 9 0 0 0 0 0 1 2;13 0 13 0 33 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1; -;4743 816 3628 2473 18 62 1552 32726 1941 286 28 18 66 102 137 116 71 1 123 1 147 0 2 0 489 2182 5;9 10 10 0 0 0 17 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 33 33 33 33 0 13 33 33 33 33 34 0 33 21 33 34 0 33 0 33 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;172 8518 777 1185 5828 3756 49 1816 1016 329 48 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 33 0 33 33 33 34 34 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;283 29 97 3 79 5683 8 16032 8 2554 18244 240 1560 7622 2548 10 15 441 17 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0;0 0 0 0 21 33 0 33 0 0 0 0 0 55 0 0 35 36 36 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 352 22419 0 82538 0 9250 1201 4555 6113 1180 477 39239 52 2 85 0 1969 1712 8595 0 29552 5045 8595 0 5158 518 8595 16 37 6 87 6 44 6 685 86 6 6508 27 82538 1567 359 2 395 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 24 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 0 1 2;0 33 33 0 0 0 0 0 0 33 0 0 33 0 0 0 0 29 30 30 0 21 22 22 0 0 0 33 0 33 0 31 0 33 0 29 30 0 13 14 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;85 26 133423 1574 486 62 46481 2 20843 538 66 0 2 0 40648 0 139737;0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 0;0 0 0 33 0 0 33 0 0 33 0 0 0 0 31 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1531 51 190 12670 532 799 30 910 4383 1758 56 3967 12 10 424 2193 0 2 0 69 108 0 1542 69 5 0 8926 31;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 0 33 34 0 0 0 33 33 34 33 34 0 33 34 0 0 0 33 34 0 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1489 1580 0 2 0 54769 1119 75 1559 5;0 0 0 0 0 9 0 0 1 2;33 33 0 0 0 13 33 34 1 2;1 1 1 1 1 1 1 1 1 1; -;4239 230 46 24254 118 22 80 11473 8517 9320 1 1 624 56642 1 1 114 204;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;21 22 0 33 0 0 0 33 33 33 0 0 33 27 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;29486 24110 67 2138 11023 1394 0 248 1123 43391 960 12617 1 912 167 1 114 5;21 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;27 28 0 33 33 33 0 33 34 34 33 34 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;842 36186 0 3300 0 32 8809 0 20299 892 755 33 1 4 9;23 24 0 0 0 0 23 0 0 0 0 0 0 1 2;29 30 0 29 0 0 29 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4952 27 4272 153 1554 6619 80479 954 7589 58 1 308 27 6326 1 18254 5;17 18 18 18 0 0 0 0 0 0 0 0 0 0 0 1 2;31 32 32 32 33 33 33 33 33 0 0 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1981 54 1557 2442 27 376 10 43375 38167 3 1119 1947 50 376 2 1628 50377;9 10 10 9 10 10 0 0 0 0 0 0 0 0 0 0 9;13 14 14 13 14 14 0 33 34 0 27 28 0 0 0 33 13;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;102 2 5356 38 34 1470 2434 819 1609 172 67 286 2560 294 343 23 1272 25 1 504 1 4 61;0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 34 13 33 34 33 33 33 0 33 33 34 33 0 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;511 422 11801 329 0 1649 278 38188 47428 1 1231 43010 0 2 0 1065 4179;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 21 22 0 0 0 33 33 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 236 217 13 103 8 184 374 3873 10 15 441 17 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 33 34 0 33 33 33 0 35 36 36 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1088 596 285 629 926 176 321 15940 220 79745 1 170 40 1 4 61;0 0 0 0 0 0 0 0 0 21 0 0 0 0 1 2;21 22 33 33 33 0 33 33 0 27 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;126 2023 1747 424 15281 12 5662 1840 59 0 566 0 3443 499 757 382;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 34 33 0 0 33 0 0 0 0 0 21 22 22 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6323 3368 76 1 988 1382 232 232 232 490 490 490 6323 3368 76 1 14 835 77 50 3 344 3263 18 16;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 0;13 21 22 0 33 33 0 0 0 0 0 0 13 21 22 0 0 33 33 34 0 21 31 32 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;418 244 946 35282 8 41948 8 103 11 578 8 214 10 20 501 1 11 217 846 1021 1 595 217 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 33 33 0 33 0 0 33 34 0 33 34 34 0 0 1 2 2 2 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;319 893 14961 3 14898 415 201 2 452 1954 2 16202 54 1167;0 0 37 0 13 14 0 0 0 0 0 9 0 0;33 33 45 0 17 18 33 0 33 33 0 13 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;516 194 664 1241 30 7952 221 126 96 306 82 27949 1591 9588 20210 83 16274 82 383 3549 83 17 992 58 12 58 2685 20210 4089 669 17 2 2 1032 674 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 34 0 33 0 0 0 0 0 33 34 34 34 0 33 0 33 34 0 0 0 33 34 34 33 33 33 33 0 0 0 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8581 57409 298 2824 36412 91 180 1 4 9;43 44 0 0 0 0 0 0 1 2;51 52 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;6823 854 3 301 0 440 1300 301;27 28 0 0 0 0 0 0;35 36 0 33 0 33 33 34;1 1 1 1 1 1 1 1; -;22 10329 12 2841 122 84 15 143 17 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;0 33 33 34 0 0 33 34 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;140 16249 5790 62 2276 66 1 4 253;0 0 0 0 0 0 0 1 2;35 36 36 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;434 484 3069 8 135 22 11 826 59 0 2 0 484 3069 3 1899 204;0 1 2 0 0 0 0 0 0 0 0 0 1 2 0 1 2;33 1 2 0 0 0 0 33 0 0 0 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7557 90 166 37312 9292 3 2694 6 23 11302 25 1751 206 1692 285 1 4 130 112;0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 1 2 2;33 0 0 33 33 0 33 0 0 31 0 33 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;105 38 656 750 3055 2994 3 2640 10 20 60 1 4 9;0 0 9 0 0 0 0 0 0 0 0 0 1 2;33 34 13 33 33 33 0 33 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 3 4172 443 183 540 373 77 804 5639 3379 126 373 369 71 1 4 9;0 0 23 24 24 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 29 30 30 0 0 33 34 34 34 0 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;309 94 3809 99 249 2701 8 93 5888 94 24 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 34 0 33 0 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;58663 1566 6021 3042 6979 244 1 23 58663 1566 6021 3042 6979 244 25 1 85770 8401 1566 912 1342 1 9956 300 5;21 22 22 22 0 0 0 0 21 22 22 22 0 0 0 0 21 22 22 5 6 0 0 1 2;27 28 28 28 33 0 0 0 27 28 28 28 33 0 0 0 27 28 28 9 10 0 33 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;773 134 2049 227 9373 6848 115 3 1495 673 8 4010 432 17 1 757 12393 1 316 100 1 316 197;0 0 0 0 9 10 10 0 0 0 0 0 0 0 0 0 0 0 1 2 0 17 18;33 0 33 0 13 14 14 0 33 0 0 33 0 0 0 33 33 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;69 761 1 567 1723 244 2656 3553 6434 120 153;0 0 0 17 18 18 17 18 18 18 18;33 33 0 31 32 32 31 32 32 32 32;1 1 1 1 1 1 1 1 1 1 1; -;6613 0 8690 0 21142 0 4855 0 12555 2 24384 0 22627 6 96033 0 119451 6 6613 0 8690 0 21142 0 4855 0 12555 1288 170 40 6 35 1288 31 95 0 1 4 202 2 4385 5522;0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 22 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 19 20 0 0 5 6 0 0 0;55 56 56 56 56 56 56 56 56 0 27 28 28 0 27 28 28 0 55 56 56 56 56 56 56 56 56 33 33 33 0 33 33 33 34 0 0 33 34 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;134 317 241 59 317 1727 1636 1351 478 3446 3 13262 99 59 1 3158 4168 2864 1 1945 3158 4168 1 1945 7 65 235 78 39 13 411;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 34 0 0 0 33 0 0 33 0 33 0 0 0 33 34 34 0 1 33 34 0 1 33 34 33 34 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;82 14630 83 206 3 8148 0 36 0 73 560 0 36 0 367 171;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 0 33 0 0 0 35 36 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13819 2476 2860 81 2223 251 447 1986 3927 4814 1563 46 81 211 5513 81 8679 0 2 0 34 5;0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 0 0 33 0 13 21 22 25 26 0 0 33 0 33 34 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5938 3632 85 1181 914 0 1181 914 39 2863 604 504 282 1 2122 2641;45 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0;53 54 54 33 33 0 33 33 0 31 0 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1108 4098 44569 11684 3 661 1 14 2103 16 1108 4098 44569 11684 15 109 80 109 1721 1 1108 4098 44569 11684 1937 661 201 1 36702 1 2548 1947 1065;11 12 12 12 0 0 0 0 0 0 11 12 12 12 0 0 0 0 0 0 11 12 12 12 0 0 0 0 0 0 5 6 6;15 16 16 16 0 33 0 0 33 0 15 16 16 16 33 34 0 0 33 0 15 16 16 16 33 33 33 0 33 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;105 38 117 97 160 4344 280 137 307 1 4577 63 4344 440 2131 0 2 0 27652 280 38051 5;0 0 0 0 0 9 0 0 0 0 0 0 9 0 0 0 0 0 1 2 2 2;33 34 33 34 0 13 33 21 33 0 33 0 13 33 33 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;26499 148 78779 376 302392 503 335 55986 3892 14 172 443 6 0 123 6 0 147 6 0 3960 16 1 111 172 443;9 10 9 10 9 10 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 0;13 14 13 14 0 0 0 13 33 0 33 34 0 0 33 0 0 33 0 0 31 0 0 31 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3116 12 502 1882 53 53 20658 0 60202 538 2302 6736 538 23 56 25 1 504 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 0 33 34 33 0 33 33 0 0 33 0 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6348 8095 141 300 6348 4220 4733 20045 2 78 2 35 19 2 41 72;19 20 0 0 0 0 0 0 0 0 0 0 0 0 1 2;25 26 33 33 27 28 33 34 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;55974 32 8740 20221 264 93 13 1298 5285 24 0 2 0 612 295 42;13 0 13 14 14 0 0 0 0 0 0 0 0 1 2 2;17 0 17 18 18 0 33 34 33 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;948 47078 4068 787 264 1 47078 4068 787 264 537 559 33456 264 1940485 5467 2598 0 2 0 111;0 31 31 32 32 0 0 31 32 32 0 0 0 0 0 21 22 0 0 0 17;33 39 39 40 40 0 39 39 40 40 0 33 21 22 0 27 28 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;80 2166 8359 3101 1136 21342 3238 8833 0;0 0 0 0 0 0 0 0 0;0 33 33 21 22 22 33 33 0;1 1 1 1 1 1 1 1 1; -;1419 798 1537 10014 0 2550 56 2033 4540 3 19 2 210 2 35 19 2 41 72;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 33 21 22 0 33 0 33 0 0 33 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8458 1661 12 703 1 125075 1 1122 1875 2 137 74 77 231 3 205 1569 2 8007 2196;19 20 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18;25 26 26 26 0 0 0 33 34 0 21 0 0 0 0 33 34 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;198 38 5126 1357 73 46 6093 75 306 5388 336 166 3047 703 1824 2 247 142;0 0 0 0 0 0 9 10 0 0 0 0 0 0 0 0 17 0;33 34 31 32 32 0 13 14 0 33 0 0 33 33 0 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 609 16 2657 149 294 23 71372 25 30 423 997 172 3 571 294 1 4 61;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 0 33 34 34 0 33 0 0 33 34 33 0 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;68854 0 219432 0 161381 18623 26337 44461 2332 290 1413 1 68854 0 219432 0 161381 18623 26337 44461 2332 290 826 37 1 3331 7 65 3703 0;0 0 0 0 0 0 0 0 9 10 0 0 0 0 0 0 0 0 0 0 9 10 0 0 0 0 0 0 17 0;33 0 0 0 0 33 31 27 13 14 33 0 33 0 0 0 0 33 31 27 13 14 33 34 0 31 33 34 31 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;31842 2928 2 2 31842 75 2928 1507 2894 652;0 0 0 0 9 10 9 10 10 10;0 33 0 0 13 14 13 14 14 14;1 1 1 1 1 1 1 1 1 1; -;82 15088 0 4115 6 0 2768 0 4115 6 0 38262 7 0 6859 3555 451 0 3606 0 473 0 2768 0 4115 232 83 0 18377 0 23108 0 10915 0 6774 0 4427 57 2634 7 0 1795 0 53925 0 2439 0 22463 0 513 0 12404 925 0 2 0 3022 43 391 361 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0;0 21 22 22 0 0 21 22 22 0 0 27 0 0 33 34 34 0 55 56 56 0 21 22 22 22 0 0 33 0 21 0 33 0 21 0 33 0 33 0 0 0 0 0 0 33 34 34 0 33 0 33 0 0 0 0 1 2 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;412 53711 897 3525;9 17 0 0;13 31 33 33;1 1 1 1; -;222 13 103 45 4898 394 572 140532 847 3 843 901 2 1126 1365 0 2 0 956 0 556 0 1157 232;0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 33 34 33 34 33 27 33 0 33 34 0 55 56 0 0 0 0 0 33 0 9 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3104 27 6485 5434 28 18 105 137 116 71 1 123 1 147 0 2 0 702 172 1270;17 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 1 0 0;13 14 31 33 33 34 33 21 33 34 0 33 0 33 0 0 0 1 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;403 1588 572 3189 8 15211 3 845 202 601 2620 202 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 33 0 33 0 33 34 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;31927 231 282 0 90 10 215 13 422 17 2 806 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 0 55 56 56 56 56 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2457 2804 11 118 6936 1282 15 143;37 38 0 0 0 0 0 0;45 46 33 34 0 0 0 0;1 1 1 1 1 1 1 1; -;102996 102996 331 2837 62 64244 178 66 147 1 123 1 3639 1 5985 1285 1 751 171 23 68 25 2 340 2659 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;15 16 16 16 0 33 0 0 33 0 33 0 33 0 33 33 0 21 22 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8177 27 62813 74 8612 238 75 581 292 2948 189 5417 59 90 236 9 3 1186 71 30 3278 59 1 8177 5;9 10 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 14 0 0 31 0 0 33 34 33 34 33 0 0 33 34 0 33 33 33 34 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 16354 1069 16 770 74 261 8 340 16354 3924 119 28 18 2 10482 355;0 0 0 0 0 0 0 0 17 18 18 18 18 18 0 0 0;0 31 0 0 33 0 33 0 13 31 0 33 33 34 0 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;963 1178 63 21 7407 0 29524 0 23059 3 13 29 123 84 1 14003 1821 12 1644 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 34 21 0 21 0 33 25 26 26 33 0 0 0 0 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;59474 1250 1184 181 50657 332 18112 1 18 69 1 45 223 5;0 0 0 0 0 0 0 0 0 0 0 1 2 2;31 32 33 0 33 0 33 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;402 397 1490 4746 15897 305 91 180 203 1 4746 15897 43 1 4 47;0 0 0 19 20 0 0 0 0 0 19 20 0 0 1 2;33 33 33 25 26 33 0 0 0 0 25 26 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 68 16 4927 19025 526 390 8 2195 7505 70 1391 103 442 1 2455 35803 100 1 110 54 115 100;0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 7 8 0 0 0 0 0 0;0 0 0 33 33 0 0 0 33 33 0 33 34 0 0 11 12 33 0 9 10 10 10;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8527 57727 2 123109 239 0 67981 0 121810 0 7896 838 1 68 942 5 0;0 0 0 0 0 0 0 0 0 0 27 28 0 1 2 2 0;33 33 0 0 0 0 21 0 21 0 35 36 0 1 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 105 38 553 20630 246 18 137 116 71 1 123 1 147 16 2 0 631 5;0 0 0 17 18 18 18 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 31 13 33 34 21 33 34 0 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;937 4064 10768 3 47;0 21 22 0 0;0 0 27 0 1;1 1 1 1 1; -;5050 459 85 29 97 12 63 135 4467 21 8 888 801 10 3108 12 240 8 85 29 97 1692 135 4467 21 1 4 79 9;0 0 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 0 0 0 0 0 0 37 0 0 0 0 0;33 33 0 0 0 33 34 0 33 0 0 33 34 0 33 0 0 0 0 0 0 33 0 33 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;529 4047 1847 995 49 5741 0 24521 48 152483 575 62 2508 159 177 3211 245 66 2 29393 2 184 17148 67 1882 25205 2 34 192 4960 31 100 0;0 0 0 0 33 34 34 34 34 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0;0 33 33 34 0 55 56 56 0 33 33 0 0 33 0 33 33 0 0 27 0 33 33 0 33 34 0 1 2 0 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;919 137 17168 22350 220 717 10459 1013 0 1627 45515 3696 23 146 26 151 25 0 1 0 558 1010 5 23 3857 7 65 25;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0;33 21 33 33 0 0 33 34 0 33 27 33 0 0 0 33 0 0 0 0 1 2 2 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 5247 16063 23176 264 16 5247 16063 23176 264 37 1 5247 16063 23176 264 104 1 5247 16063 23176 264 87 0 2 427 262 318 5;0 23 24 24 24 0 23 24 24 24 0 0 23 24 24 24 0 0 23 24 24 24 0 0 0 0 0 1 2;0 29 30 30 30 0 29 30 30 30 33 0 29 30 30 30 33 0 29 30 30 30 31 0 0 33 31 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5924 799 2091 501 3 1851 176 3376 4251 2 2 5924 2714 161 2 2 5924 253 2 2 34 5924 5 59;11 0 0 0 0 0 0 0 0 0 0 11 0 0 0 0 11 0 0 0 1 2 2 0;33 0 0 0 0 33 0 33 34 0 0 33 21 22 0 0 33 33 0 0 1 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1407 12687 12746 1372 51 2563 1489 289 6619 19962 45228 81 1344 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 21 22 22 0 33 33 0 33 21 22 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;402 7608 7428 32 7428 146 1902 10 13 124 3 442 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 34 0 33 0 0 33 34 34 34 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15477 10189 21559 1959 46 3441 95192 82 20656 83 37203 28567 1 268 1 279 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 33 34 0 33 0 0 33 0 33 34 0 21 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3584 2342 30202 28473 638 1 3584 2342 30202 638 39 28473 603 39 57 535 367 1 3584 2342 30202 40291 0 2 0 5361 638 18807 7 65;0 0 0 0 0 0 19 20 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 34 0 33 0 33 34 34 33 0 0 0 0 0 0 33 0 33 34 34 0 0 0 0 33 33 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1971 54918 0 23 2319 23701 3389 0 17552 90178 25 16 23 1264205 0 54918 0 23 2319 23701 3389 0 17552 90178 25 25 307 1 44 1 927 1 1396 1 100 1 1971 54918 0 23 2319 23701 3389 0 17552 90178 25 12288 307 2 4257 1911;0 0 0 0 0 0 31 32 0 0 0 0 0 0 0 0 0 0 0 0 31 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 31 32 0 0 0 0 0 0 0 19 0;0 33 0 0 0 33 39 40 0 0 0 0 0 0 0 0 0 0 0 33 39 40 0 0 0 0 0 33 0 33 0 33 0 0 0 33 0 33 0 0 0 33 39 40 0 0 0 0 33 33 0 25 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1729 20031 8 33175 12009 8 999 96 1956 462 7104 8 93 13184 5751 442 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;0 45 0 33 33 0 0 0 33 33 34 0 0 33 33 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 14201 2291 6 88 14201 2291 2592 16 2 88 247 2578 5;0 9 10 0 9 9 10 0 0 0 9 1 2 2;0 13 14 0 13 13 14 33 0 0 13 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1425 3122 1303;0 0 0;33 33 33;1 1 1; -;16375 1697 358 0 1537 1724 1314 40 23 312 213 1724 6 37141 1724 6 10014 1724 6 14443 1724 25 6 8590 51 357 7759;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0;21 22 22 0 33 34 33 33 0 33 34 34 0 33 0 0 33 0 0 33 0 0 0 33 0 13 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;41682 7 16808 1921 7 1054 0 2 0 10166 0 151559 0 4208 9076 0 23926 0 31938 0 18447 60;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 21 33 33 34 0 0 0 33 0 21 0 21 22 0 33 0 55 56 56 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 963 331 10 236 7291 8 134 160 127 248 122 393 21 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;0 33 33 34 0 33 0 0 0 33 34 33 34 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3094 1229 86 2763 1348 1240 15 347 1 4 9;5 6 6 0 0 0 0 0 0 1 2;9 10 10 33 33 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;30804 46 9992 3121 136 49 1685 4681 3 90 48 8 699 916 10396 3401 2 45 81 5;21 0 0 0 0 33 34 34 34 34 34 0 0 21 22 22 0 1 2 2;27 0 0 0 0 25 26 26 26 26 26 0 33 27 28 28 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;64 13 614 240 2670 3 2370 3 12440 6474 289 13 29 240 978 879 3114 6 240 312 2370 2688 106 2370 6 179 62 66 2370 6 509 240 978 879 77 122 11 29 772 7275 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 0 0 0 33 0 33 0 33 0 33 34 0 33 34 34 33 0 0 0 33 0 35 36 0 0 0 0 33 0 0 33 34 34 0 0 0 0 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6689 194 3719 501 2581 1213 640 504 30633 1 33 505 31 1 33 505 31 2 624 463 5 107 344 372;37 38 38 38 0 0 0 0 37 0 1 2 2 0 1 2 2 0 0 0 0 0 0 0;45 46 46 46 33 33 33 0 45 0 1 2 2 0 1 2 2 0 33 34 34 33 21 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;29232 1214 773 903 134 249 91 180 2852 17 710 10 35872 432 17 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 33 0 0 0 0 0 33 0 33 34 33 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;45 18851 672 45904 49116 0 34 142 414 2888 962 1039 426 269 0 1 790 12761 45 18851 672 2836 62 67335 672 66 0 1 790 12761;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 33 33 33 0 21 22 33 0 0 21 22 22 0 0 1 2 33 34 33 33 0 29 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1015 140 554 878 1827 740 6 1015 140 1827 6 878 26 2085;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 33 33 34 33 0 33 34 33 0 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5856 8 1333 249 1502 361 84 1 4 9;0 0 0 0 0 0 0 0 1 2;33 0 33 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;1539 3033 22451 929 20 356 93 21666 1 4 9;0 0 23 24 0 0 0 0 0 1 2;33 33 29 30 33 34 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;88 2194 6463 719 1830 91 1 1183 1 9018 8 2846 719 8 2820 719 0 2 0 1774 884 224 23 13 199 712 729 1256 25;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0;13 33 34 33 34 0 0 33 0 33 0 33 34 0 33 34 0 0 0 1 2 2 0 33 34 33 33 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4779 0 28319 0 2 0 6774 0 499585 0 2392 0 3170 0 4513 0 73588 0 1977 0 519 0 9783 0 21542 0 9413 0 7074 0 1147 0 1682 0 513 0 24050 0 7365 0 104738 0 169818 0 9778 0 12261 0 39417 0 103780 0 6238;0 0 0 0 0 0 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 0 51 0 0 0 21 0 0 0 33 0 33 0 33 0 33 0 33 0 33 0 25 0 33 0 33 0 33 0 33 0 33 0 33 0 21 22 22 0 21 0 0 0 33 0 0 0 21 0 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;47319 0 9815 0 2 0 1066 0 19150 0 85 0 16260;0 0 0 0 0 0 0 0 23 24 24 0 0;21 22 22 0 0 0 33 0 29 30 30 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;2369 5606 1 154735 1 187 19367 1 277 0 2 0 117 251 856 6916 7141 5729 516 58;0 0 0 39 0 0 0 0 0 0 0 0 0 0 0 39 40 40 40 40;33 33 0 47 0 0 0 0 0 0 0 0 0 0 33 47 48 48 48 48;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;89947 23015 46 12 2046 3903 3 3522 0 2 2 0 479 55 0 2 2 0 34 119 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 34 0 33 34 33 0 33 0 0 0 0 0 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;402 214 10 20 1232 17 781 59 11 42 165 10 19992 358 8 4 136 21 136 8 12 1091 84 1 4 9;0 0 0 0 29 0 0 0 0 0 0 0 29 30 0 17 0 0 0 0 0 0 0 0 1 2;33 33 34 34 33 0 33 0 33 34 34 0 37 38 0 31 0 0 0 0 21 22 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11 89 11 5240 2518 430 6078 2376 3 607 4495 2611 2470 8 27980 2800 3 94 17 5240 77 33 8 22 699 136 19 8 12 96 183 3 84 507 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 21 33 0 21 33 0 0 33 33 34 0 0 33 0 33 0 21 33 34 0 33 34 0 33 0 33 34 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2756 2800 183 0 36 0 31219 67217 0 3117 2419 0 10300 2800 183 0 79697 0 13390 79697 0 13390 10300 0 2756 0 17738 0 334 128;17 0 0 0 0 0 0 0 0 0 0 0 23 24 24 0 0 0 0 0 0 23 24 0 17 0 0 0 0 0;31 33 34 0 0 0 33 34 0 33 34 0 29 30 30 0 33 0 21 22 0 29 30 0 31 0 31 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4468 120 149 0 193 602 1336 138 343 0 2 0 421 499 480 480;0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;33 34 34 0 33 34 33 34 33 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;25414 195 514 3 27275 2481 0 2 0 364 5;0 0 0 0 13 0 0 0 0 1 2;33 33 34 0 17 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;7181 46 238 27 37336 0 6559 2953 50624 2797 10455 7 844 764 1 268 1 279 5;0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 0 1 2;33 0 33 34 33 0 33 0 31 0 33 0 55 56 0 21 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;934 4053 122 749 547 3320 211;0 0 0 0 0 0 0;33 34 33 34 33 33 33;1 1 1 1 1 1 1; -;598 27 18276 351 697 192 359;17 18 18 18 18 18 18;13 14 33 0 0 33 34;1 1 1 1 1 1 1; -;397 27 772 56 11 690 11 6713 19588 3 1512 866 8 1361 350 614 453 37 3 12 10 6713 3 1512 866 514 8 5713 11 122 6713 84 1 4 9;0 0 0 0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0 1 2;33 33 34 0 0 0 0 33 33 0 29 30 0 0 0 0 0 33 0 33 34 33 0 29 30 0 0 33 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;210508 21808 3 47;0 0 0 0;0 33 0 1;1 1 1 1; -;1746 3357 21517 850 682 3466 6135 16149 0 10880 62356 15833 17492 3448 6563 4679 1 207 5138 2 1397 1 1044 980 1 114 5;0 0 39 0 0 0 0 0 0 17 18 0 21 22 0 0 0 0 0 0 9 0 0 0 0 1 2;0 0 47 33 34 0 0 33 0 31 32 33 27 28 33 33 0 33 33 0 13 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;250 25909 5173 2897 5986 120 532;0 0 0 0 0 0 0;0 0 33 33 33 34 34;1 1 1 1 1 1 1; -;43417 49 14126 18767 48 5354 176 273 62 628 66 1 5681 483 273 1 6945;27 27 28 28 28 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 34 0 33 0 33 0 0 0 0 33 34 33 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2324 115 438 15723 6101 0 2 0 39 13 9936 5 0 956 0 556 0 131619;0 0 0 0 0 0 0 0 1 2 2 2 0 0 0 0 0 0;21 22 33 33 34 0 0 0 1 2 2 2 0 0 0 33 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;18505 14423 1 18505 14423 37 1 410 18505 14423 104 26 182 0 2 0 111;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17;33 33 0 33 33 33 0 33 33 33 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2656 934 23 170197 25 2 8518 1878 1 22648 243 1 37418 10419 418 0;17 18 0 0 0 0 0 0 0 0 0 0 5 6 6 0;31 32 0 21 0 0 0 33 0 21 33 0 33 34 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4244 229 133 27023 133 32 4916 36163 133 1903 30 20 676 103 3906 0 2 0 612 295 42;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;33 33 0 33 0 0 21 22 0 33 0 33 34 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1073 46 14457 1557 1297 376 82 65282 83 0 283 27 13 75 89 21 82 1557 83 1 342 5 0;9 0 9 10 10 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;13 0 13 14 14 14 0 33 0 0 0 0 33 34 35 36 0 0 0 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11 15818 8 15 441 432 1 896 5512 336 43 1 4 47;0 0 0 0 0 0 0 23 24 24 0 0 1 2;0 33 0 0 33 0 0 29 30 30 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5594 11989 53646 334 398 199 1 64165 0 91679 0 368 0 69506 1 5594 11989 53646 300 128 0 2 0 6129 43 5594 11989 53646 1311;45 46 0 0 0 0 0 0 0 0 0 0 0 0 0 45 46 0 0 0 0 0 0 0 0 45 46 0 0;53 54 0 21 22 0 0 9 0 33 0 33 0 53 0 53 54 0 33 33 0 0 0 33 0 53 54 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;157 46465 290 0 2 0 3094 253;9 10 10 0 0 0 1 2;13 14 14 0 0 0 1 2;1 1 1 1 1 1 1 1; -;3169 29148 31 3399 732 0 5174 544 17536 2044 267 2057;9 0 0 0 0 0 0 0 0 0 0 23;13 33 33 34 34 0 33 34 33 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1; -;564 113 1 683 5373 100 1 6225 5016;9 10 0 9 0 0 0 1 2;13 14 0 13 33 34 0 1 2;1 1 1 1 1 1 1 1 1; -;4762 26 2083 0 856 2980 2 102 0 2861 4093 199 1780 267 22876 2 416 40 2 416 71 2 416 71 5;0 0 0 0 0 0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0 1 2 2;33 34 34 0 33 33 0 33 0 33 21 22 29 30 0 0 33 33 0 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;295 3 4072 74 748 67 3047 435 1 504 1 4 61;0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 33 0 33 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;2 45176 255;0 17 18;0 31 32;1 1 1; -;576 41876 686 1170 339 398 170 979 1836 100;9 0 0 0 0 0 0 0 0 0;21 33 33 34 33 33 33 33 33 33;1 1 1 1 1 1 1 1 1 1; -;68734 62 4243 2445 178 66 1 4 253;9 10 10 10 10 10 0 1 2;31 0 13 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1; -;42602 0 81098 3 699 0 36 0 7660 62 7368 66 0 2 0 775 210 8 801 453 33;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 0 27 0 33 0 0 0 33 0 9 0 0 0 0 33 33 0 33 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4762 1081217 10619 456 138 23403;0 0 27 28 28 28;33 0 35 36 36 36;1 1 1 1 1 1; -;461 16039 3657 15 124 1 2087 5;9 17 0 0 0 0 1 2;13 31 33 0 0 0 1 2;1 1 1 1 1 1 1 1; -;14940 3780 199 63 104 269 8217 91 180 1 4 9;0 0 0 0 0 0 0 0 0 0 1 2;13 33 34 0 33 34 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;650 27 384 594 23 1364 25 186 640 189 20475 1 4 61;17 18 18 0 0 0 0 0 0 0 0 0 1 2;13 14 33 0 0 0 0 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 447 0 87 2001 0 352 0 2397 76 0 675 12108 86 36264 165572 0 12108 86 0 333 2102 16 37 6 87 6 44 6 12108 86 6 6726 27 2489 31830 1067 110 1097 28 18 2 395 5;0 9 0 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18 18 18 18 18 18 18 18 0 1 2;0 13 0 31 33 0 33 0 33 0 0 0 33 34 33 0 0 33 34 0 0 33 0 33 0 31 0 33 0 33 34 0 13 14 0 0 0 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;789 49372 394 1896 28 18 36 49372 394 1896 36 789 49372 36 49372 1896 36;17 18 18 18 18 18 0 0 0 0 0 9 0 0 0 0 0;13 27 33 34 33 34 0 27 33 34 0 13 27 0 27 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;69274 0 26994 2 151 3840 8031 177 36144 129 94757 200 583 200 278 14560 7585 602 1 405 1 35 19 31 95;23 0 0 0 0 0 23 0 0 0 0 0 0 0 0 23 24 0 0 0 0 0 0 0 0;29 0 33 0 33 21 29 0 31 0 21 33 34 21 22 29 30 33 0 1 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2350 106 3232 7410 3070 3 130 5968 108751 10271 2896 208 2 14 1030 5 16 2 952 252 31 438 696 211 2 163 7 1058 7 65;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 33 33 0 33 33 33 33 34 33 0 0 1 2 0 0 33 0 33 33 33 33 0 33 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;16218 0 368 0 17097 0 22601 0 683481 38357 0 2 0 3107;0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 0 33 0 21 0 33 0 0 21 0 0 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;55657 0 28591 1 5286 2139;0 0 0 0 0 0;33 0 0 0 33 33;1 1 1 1 1 1; -;441202 0 569878 0 56289 41641 0 413311 0 14410 0;0 0 0 0 0 0 0 0 0 0 0;33 0 0 0 27 21 0 21 0 0 0;1 1 1 1 1 1 1 1 1 1 1; -;7061 3010 1033 39 479 1997 856 53 53 1207 3 361 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 33 34 0 33 33 34 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13693 225 96 511 145 1868 782 3 19751 813 1 4 9;0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 0 0 0 33 0 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;3403 46 14 73 17727 16 836 358 1794 335 3568 2717 1 73 17727 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0 1 2;33 0 0 25 26 0 33 37 38 0 33 33 0 1 2 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13618 3448 21600 1 44 1 709 253;29 30 30 0 0 0 1 2;37 38 38 0 33 0 1 2;1 1 1 1 1 1 1 1; -;5606 9670 9303 1145 2166 242 1 4 61;0 0 0 0 0 0 0 1 2;33 33 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1 1; -;150454 3 51786 877 1 3177 406 396 1 5687 1683 1708 7144 5 0;0 0 0 0 0 0 0 0 0 1 2 2 2 2 0;33 0 33 33 0 33 33 33 0 1 2 2 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2510 56 3 18708 11 638 24 30 215 70 136 0 2722 3635 13 195 127 22836 14951 1948 3309 1 2442 54 2859;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 22 22 0 33 0 33 34 0 0 0 33 33 34 34 34 33 33 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;200 206 2180 20 3 16059 1 4 9;0 0 0 0 0 0 0 1 2;33 34 34 0 0 37 0 1 2;1 1 1 1 1 1 1 1 1; -;6899 753 494 1626 87 1 6899 753 494 1626 87 37 1 6899 753 494 1626 87 104 26 182 0 2 0 111;23 24 24 24 0 0 23 24 24 24 0 0 0 23 24 24 24 0 0 0 0 0 0 0 17;29 30 30 30 31 0 29 30 30 30 31 33 0 29 30 30 30 31 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;357 76511 537 366 741 259;9 0 0 0 0 0;13 33 0 33 34 0;1 1 1 1 1 1; -;284 984 113 25687 148 63 958 29336 148 3 1892 1 4 9;0 9 10 9 10 0 9 9 10 0 0 0 1 2;0 13 14 13 14 0 13 13 14 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2293 5710 45 700 1001 11 39287 177 2 8 56 1313 1474 177 7 7 7 1 4 79 9;37 38 0 0 0 0 37 0 0 0 29 30 30 0 0 0 0 0 0 0 0;33 34 0 33 33 0 45 0 0 0 37 38 38 0 0 0 0 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1931465 0 7770 1931465 467 99 585 94 730 761 2 14 224 99 43 16;0 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 13 0 33 0 0 33 34 33 0 0 0 0 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;9185 32 943 158 219 2054 17 1 4 9;0 0 0 0 0 0 0 0 1 2;33 0 53 54 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1; -;14 35993 17219 3 661 16 35993 17219 15 109 1 35993 17219 3 1937 661 1 103 2231;0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 0 0 0;0 33 33 0 33 0 33 33 33 34 0 33 33 0 33 33 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;155 20 415 58 5419 9499 1 4 209 132;0 0 0 0 0 0 0 17 23 24;0 0 0 0 33 33 0 31 29 30;1 1 1 1 1 1 1 1 1 1; -;4292 29229 66257 1 4292 29229 66257 37 1 410 4292 29229 66257 104 26 182 0 2 0 111;0 39 0 0 0 39 0 0 0 0 0 39 0 0 0 0 0 0 0 17;21 47 33 0 21 47 33 33 0 33 21 47 33 33 0 33 0 0 0 31;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4833 73 4851 4198 23 12462 178 25 906 1065 1 12462 45 873 5036 99 15160 258 1715 108 129 1324 1065 1 88 2 588 648 5;11 12 17 18 0 9 0 0 0 0 0 9 10 10 0 0 9 10 10 10 0 0 0 0 9 0 1 2 2;15 16 31 32 0 13 0 0 33 33 0 13 13 14 33 0 13 14 33 34 0 0 33 0 13 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;3906 26077 194 7869 3 903 319 266 1 2582 372 1 22966 5 882 440;37 38 38 38 0 0 0 0 0 0 0 0 1 2 0 0;45 46 46 46 0 0 33 33 0 33 33 0 1 2 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1532 7091 0 39355 577 752 15 577 1 4 9;23 24 0 0 0 0 0 0 0 1 2;29 30 0 0 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;32227 0 2 0 3107;0 0 0 0 1;33 0 0 0 1;1 1 1 1 1; -;92 33293 2401 538 14194 4210 2077 2 496 14194 4210 1 2113 14194 4210 1 14194 4210 37 1 14194 4210 267 1 5366 3907 1 5366 3339 1 92 33293 7261 1355 28 18;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 31 32 0 17 18 18 18 18 18;13 35 35 36 21 22 21 0 21 22 22 0 33 21 22 0 21 22 33 0 21 22 33 0 33 33 0 39 40 0 31 32 32 32 32 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;340 27 225 11 5633 3 0 14349 308 1 4 9;9 10 0 0 0 0 0 0 0 0 1 2;13 14 0 0 33 0 0 33 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1; -;3527 27 3042 5272 33324 75 281 850281 259 371;9 10 9 10 10 10 0 0 0 0;13 14 13 14 14 14 0 0 33 34;1 1 1 1 1 1 1 1 1 1; -;499 840 389 40149 19581 9381 238 18854 0 152829 50159 19762 2371 488 1 841 1122 2 6182 1 114 1044 980 1 114 5;0 0 9 0 0 17 0 0 0 21 0 0 0 0 0 0 0 0 17 0 17 0 0 0 1 2;33 34 13 0 33 31 0 33 0 27 33 33 0 0 0 33 33 0 47 0 31 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 324 6822 46 21925 2825 3778 19182 1969 335 137 1327 6 22 324 6822 46 21925 2825 3778 19182 1969 335 534 724 40 6 869 367 0 2 0 9324 1635;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;35 36 36 0 35 36 36 36 36 36 21 33 0 35 36 36 0 35 36 36 36 36 36 33 33 33 0 33 33 0 0 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 1327 16 127 4185 2206 7471 0 2392 7 378 152 26668 3 667 4911 0 170 31 367 1 8566 301 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;0 33 0 33 34 33 33 0 33 34 33 0 33 0 33 34 0 33 33 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 504 16 5239 10631 7870 446 1316 471 7 5867 353 332 0 8617 85 429 1 69 108 1 2633 5;0 0 0 7 8 8 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 0 11 12 12 33 34 33 0 33 0 0 0 33 0 0 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;126 69 89 11 2321 254 16523 1112 19838 17 2321 3 134 10 7480 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 33 34 33 0 33 34 33 0 33 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;330 46 25276 1211 21622 0 181 77 179 8033 306 5221 220 2 151 21598 625 1 452 131 1 342 5 0;9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0;13 0 47 0 33 0 0 33 34 34 0 33 0 0 33 31 32 0 33 33 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 115904 48 14680 3137 806 2253 1 2 806 2 35 19 2 41 72;19 20 20 0 0 0 0 0 0 0 0 0 0 0 1 2;25 26 26 29 30 33 33 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 878 1318 85 48 78 888 2279 1194 671 266 1 282 4174;45 46 46 46 46 0 0 0 0 0 0 0 0 0;53 54 54 54 54 33 34 33 33 33 34 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;7842 1833 46 557 8994 541 68651 136 49773 1 188 69 108;0 0 0 0 0 0 0 0 0 0 17 18 18;33 34 0 0 0 0 33 0 33 0 9 10 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;3951 1342 1767 1333 3522 921 0 702 1904 136 38559 21 1 1067 363 3951 1342 19 199;45 46 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0;53 54 33 33 33 33 0 1 0 0 33 0 0 0 0 53 54 33 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;10146 7362 3 1937 661 60 1 4 9;11 12 0 0 0 0 0 1 2;15 16 0 33 33 0 0 1 2;1 1 1 1 1 1 1 1 1; -;6355 367 3731 4868 12517 731 954 62 106 66 1 6355 1 811 3937 381;0 0 0 0 0 0 0 0 0 0 0 0 0 9 17 18;21 22 33 33 21 22 33 0 0 0 0 33 0 13 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1454 50 14234 19634 3 1393 765 11 13 29 1091 590 13 124 3 1455 3973 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 34 33 0 0 33 0 33 34 0 0 0 0 0 33 33 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;198 38 154 97 146 181 472 145 5548 246 1736 3 79 606 2924 335 798 1 4 79 9;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 33 34 34 34 33 0 33 0 33 0 21 33 0 0 33 0 21 22 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;11809 0 509 2608 10 20 156 60;0 0 0 0 0 0 0 0;33 0 0 33 33 34 33 0;1 1 1 1 1 1 1 1; -;80 306 184 3 1169 5 1133 11 213 655 249 677 267 1 4 815;0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;0 0 33 0 33 34 0 0 0 33 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 10877 48 0 197081 0 7314 503 3787 0 85 0 81 115 81 852 1 10877 1 19 1 1337 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 0 0 0 0 0 33 0 0 0 33 34 0 0 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;102 38 151 97 630 181 0 3228 283 1 3427 1006 1 39 2744 55 51 483 15398 251 1311 1 3427 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 34 34 0 25 26 0 35 36 0 0 21 0 0 0 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;22 3 158 73 324 73 1113 18853 1 19 31 95 2 41 72 712;0 0 0 0 0 0 0 0 0 1 2 0 0 1 2 0;33 34 34 0 0 0 33 0 0 1 2 33 0 1 2 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4173 436 586 80 31177 1 4 61;31 32 0 0 0 0 1 2;39 40 33 0 33 0 1 2;1 1 1 1 1 1 1 1; -;3187 11281 67 6444 11281 30 5391 56 11 20 12 453 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 21 22 0 33 0 33 34 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;101 57175 973 8564 3 156 1 2597 10 20 1 1033 1576 1 4378 1 3123 1 4942 1 2527 9561 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2;21 22 22 22 0 33 0 21 33 34 0 33 33 0 33 0 33 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15337 3961 3100 3 1756 2553 176 15337 5694 3626 1 609 1 4 61;0 37 38 0 0 0 0 0 0 0 0 0 0 1 2;33 45 46 0 33 34 0 21 22 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2689 1156 16623 1 504 1 4 61;0 0 0 0 0 0 1 2;33 21 22 0 0 0 1 2;1 1 1 1 1 1 1 1; -;330 4085 1746 198 417 630 53 431 186 1103 28499 628 304 5676 168 1029 2 2 140 0 2 0 421 499 480 480;17 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 2;31 32 0 33 0 33 0 33 0 33 0 33 34 33 34 33 0 0 33 0 0 0 1 2 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 0 10617 60325 44 16 1 8519 281 1 4279 1 247 110 5;0 0 23 24 0 0 0 9 0 0 0 0 1 2 0;0 0 29 30 33 0 0 13 0 0 33 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;155218 320 451 0 91442 0 32340 0 44566;0 0 0 0 0 0 0 0 0;21 0 0 0 33 0 21 22 22;1 1 1 1 1 1 1 1 1; -;1480 113 1905 1362 98 3174 718 198 691 2655 103 2265 39 200 393 878 168 521 23 2476 25 1029 0 7551 55 1611 361 1 4 61;0 0 17 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 14 31 32 32 32 32 33 0 33 55 56 33 34 0 33 33 33 0 0 0 33 0 9 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 3806 2211 623 78 76 267 15 690 48 1 4 9;0 0 0 0 0 0 0 0 0 0 0 1 2;0 33 29 30 30 30 33 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;399 4038 5228 3312 119 28 18;17 18 18 18 18 18 18;13 0 0 33 33 33 34;1 1 1 1 1 1 1; -;895 113 4305 27 32829 75 81 9910 3803 2150 99 8 34 2172 7332 237 4305 32829 2967 3 147 0 2 0 4305 469;9 10 9 10 9 10 9 10 10 0 0 0 17 18 18 18 18 18 18 0 0 0 0 0 9 0;13 14 13 14 13 14 21 22 22 33 0 0 31 32 32 32 13 13 33 0 33 0 0 0 13 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4041 0 27489 149 787 687 8123 3 671 2 149 54 115;5 6 6 0 0 0 0 0 0 0 1 2 2;9 10 10 33 34 33 33 0 33 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;963 11 1924 94 3 678 6603 24 17 6598 1924 94 3 678 59 1 789 43 1 4 47;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 1 2;33 0 0 33 0 33 33 0 0 0 0 33 0 33 0 0 1 2 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2593 56 1132 3 3189 520 2205 4021 0 6043 42 997 17 1 6530 10425 0 2 0 6530 5;0 0 0 0 0 47 48 48 0 19 20 0 0 0 0 0 0 0 0 1 2;33 33 34 0 33 41 42 42 0 25 26 33 0 0 33 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;780 26909 6845 1 4 61;0 0 0 0 1 2;33 34 34 0 1 2;1 1 1 1 1 1; -;281 353 101 23 88 25 71 161 28 18;0 0 0 0 0 0 0 0 0 0;0 0 0 0 13 0 33 34 33 34;1 1 1 1 1 1 1 1 1 1; -;94 15981 1558 89 1077 21 8 8 11 1333 9 15 441 24 1 896 829 9064 43 1 4 47;23 24 0 0 0 0 0 0 0 0 0 0 0 0 0 23 24 24 0 0 1 2;29 30 33 0 33 0 0 0 0 33 33 0 33 0 0 29 30 30 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;4995 299 92 35726 4588 1 761 5;9 0 17 18 0 0 1 2;13 33 31 32 33 0 1 2;1 1 1 1 1 1 1 1; -;49 46427 8 79688 11 1680 59 48 14476 19194 30 2389 0 23 1917 15224 0 242 1004 12232 0 38 103 0 8077 30432 4225 1604 55923 16447 1346 0 488 1346 0 25884 25 7 724 40 0 4 271 0 4 1429 40 1 787 4672;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 5 6 0 0 0 0;0 33 0 33 0 33 0 0 33 37 0 0 0 0 33 33 0 33 34 33 0 33 34 0 33 33 34 0 33 33 0 0 0 0 0 33 0 0 33 33 0 1 2 0 33 34 33 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;203 2 0 1242 1164 0 2 0 605;0 0 0 0 0 0 0 0 1;0 0 0 33 34 0 0 0 1;1 1 1 1 1 1 1 1 1; -;35261 0 20686 320 451 0 10528 0 11622 0 271029 53 31 291 53 387 5 8 19 35 31 95;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2 0 0 0 0 0;31 0 21 0 0 0 33 0 0 0 0 0 21 22 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;198 390 25022 129 52 7 117 2542 425 91 99 675 1 4 9;0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 0 11 12 0 0 0 33 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;561 3018 161 150 120 1051 0 4734 5447 1051 1 4 61;37 38 0 0 0 0 0 0 0 0 0 1 2;33 34 34 34 33 34 0 33 33 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;9716 3658 4792 115789 6 1432 2858 2636 772 0 1432 2858 13288 178 0 9716 3658 4792 115789 37 14 92 1721 6847 3 9716 3658 4792 115789 225 155 16 11055 73 5112;0 11 12 12 0 17 18 18 18 0 0 0 0 0 0 0 11 12 12 0 0 9 0 0 0 0 11 12 12 0 0 0 0 0 0;37 15 16 16 0 31 32 32 32 0 0 0 33 0 0 37 15 16 16 33 0 13 33 33 0 37 15 16 16 33 34 0 33 34 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;284 2 85 8 2 52 8 2 57 8 228 8 129 214 283 29 549 45 4273 19151 13 29 549 905 121 239 8 239 3 673 6440 10 12 9332 643 3 1540 8 455 30 969 1141 265 3 39173 9590 1929 450 3 5588 10 1 361 1 4 554;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 34 33 33 33 34 0 0 0 0 0 0 0 0 0 0 21 22 22 22 22 0 0 0 33 0 0 0 33 33 34 0 0 33 0 0 33 0 21 22;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 2674 2259 46 20365 48 357 241 591 0 574 522 7411 1087 179 1 188 167;19 20 20 20 20 20 9 0 0 0 0 0 0 0 0 0 17 0;0 25 26 0 33 0 13 0 0 0 0 0 33 33 34 0 31 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;26126 854 3363 0 53794 2419 1759 0 44420 0 220 904 0 14 1776 0 6165 22 31967 0 1598 0 4540 2 10953 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 33 34 0 27 28 33 0 47 0 0 0 0 0 0 0 33 34 34 0 55 0 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14439 46 4385 385 3 492 915 82 12 5680 860 16259 83 1 167 1 279 5;21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;27 0 0 33 34 34 33 0 33 34 34 34 0 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;1648 2716 227 21 1928 1202 3 12605 80 793 1 4 9;0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 0 0 0 0 0 33 0 33 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;271 4852 3186 602 5682 1151 639 0 2 0 918 0 474 0 759;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;0 9 21 22 33 34 33 0 0 0 33 0 33 0 1;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 331 628 523 16 39936 0 36 0 451 11015 2916 12 36190 0 36 0 10661 78;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 18;0 25 26 26 0 33 0 0 0 0 25 26 0 33 0 0 0 31 32;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;36210 34111;23 24;29 30;1 1; -;32020 10126 26046 711 2937 6930 105 2501 9544 838 170 40 1 146099 3949 8119 44 138 926 1 14 392 2093 16;0 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 33 34 33 34 33 33 29 21 22 33 0 0 33 33 33 33 33 0 0 21 22 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;6636 3 358 4654 3 185 1 185;0 0 29 30 0 1 0 1;33 0 37 38 0 1 0 1;1 1 1 1 1 1 1 1; -;1281 963 3154 30 34 219 2775 465 1 4 9;0 0 17 0 9 0 0 0 0 1 2;0 33 31 21 22 0 33 34 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;141 45 13914 6065 3 4712 1 4 61;0 0 0 0 0 0 0 1 2;33 0 21 22 0 33 0 1 2;1 1 1 1 1 1 1 1 1; -;22 30 4052 4683 8 381 93 2242 442 17 2 2475 229;0 0 0 0 0 0 0 0 0 0 0 0 0;33 34 33 33 0 33 0 33 33 34 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1; -;836 264 201 1 23593 32 2776 619 2615 39 22149 224 2 546 2 35 19 2 41 72;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;33 34 33 0 21 22 22 22 22 0 33 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;13 29 6992 13 29 115 817 20 1 4 9;0 0 0 0 0 0 0 0 0 1 2;33 34 33 34 0 0 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1; -;146 407 82 7833 83 344 690 1648 4476 0 317 5586 42 3132 0 82 0 4488 1986 0 83 1 1116 258 1833 1 4488 5;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0 0 1 2;0 0 0 33 0 21 0 33 34 0 0 33 0 33 0 0 0 13 33 0 0 0 35 36 36 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;15000 6884 10 1322 2989 6 13316 240 10 4589 2989 1361 3061 3 10 91 1840 8 17 17 1 4 209 132;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 23 24;33 34 0 33 33 0 33 34 0 33 33 0 33 0 0 0 0 0 0 0 0 31 29 30;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;40468 0 16577 0 2222 1760 1 4 130 112;21 0 0 0 0 0 0 1 2 2;27 0 33 0 33 34 0 1 2 2;1 1 1 1 1 1 1 1 1 1; -;92 2993 1121 310 919 2 315 2 35 19 2 41 72;9 27 28 0 0 0 0 0 0 0 0 1 2;13 35 36 33 34 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1; -;1089 314 652 11070 198 1103 113 1609 314 41111 3742 30735 1853 14719 3575 3247;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;33 33 0 33 33 33 33 34 33 33 33 33 34 0 33 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;8935 6519 76 748 1274 68 45 25604 3 921 1090 2142;23 24 24 0 0 0 0 0 0 0 0 0;29 30 30 33 34 0 0 33 0 33 33 33;1 1 1 1 1 1 1 1 1 1 1 1; -;5258 237 4754 11 20 416 1 4 9;0 0 0 0 0 0 0 1 2;33 21 22 33 34 33 0 1 2;1 1 1 1 1 1 1 1 1; -;2778 2529 45 80 7628 9700 17 17 1 2778 43 1 4 47;23 24 0 0 0 0 0 0 0 17 0 0 1 2;29 30 0 0 33 33 0 0 0 31 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;57 910 5728 1212 45 43783 3 470 549 3092 366 470 549 3 162 246 54 162 60 6010 3 470 549 10 6999 32 19137 470 549 3 91 2471 60 1 4 130 112;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 1 2 2;0 0 33 34 0 33 0 33 34 0 0 33 34 0 33 34 34 34 0 13 0 33 34 0 33 0 33 33 34 0 0 0 0 0 1 2 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;303 262 263 1229 86 26 262 10247 67 31697 4062 229 507 0 2 0 199654 17566 3 2816 0 2 0 188 204;0 5 6 6 6 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 5 6;33 9 10 10 10 0 9 10 0 9 33 33 33 0 0 0 0 33 0 33 0 0 0 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;49 779 586 3884 2866 416 48 1 1197 1735 1388 248 1005;0 0 0 0 0 0 0 0 0 0 0 0 0;0 33 34 34 33 33 0 0 33 33 35 0 0;1 1 1 1 1 1 1 1 1 1 1 1 1; -;218 3937 15382 7232 256765 381 241 15382 917 8 8092 647 19 59 2 458 2 35 19 2 41 72;17 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 2;13 31 29 33 0 33 0 29 0 0 21 33 33 0 0 33 0 21 22 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;2343 3445 3445 618 20209 604 15 254 17 0 2343 3445 3445 618 20209 604 282 2 1053 345 5 23 24618 7 73158 4667 7 2171 25;45 46 46 46 0 0 0 0 0 0 45 46 46 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;53 54 54 54 0 0 33 34 0 0 53 54 54 54 0 0 33 0 33 33 34 0 33 0 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 592 76 21394 15307 16 1 592 76 21394 15307 37 1 592 76 21394 15307 104 1 592 76 21394 15307 87 1 395 5 0;0 0 0 0 17 0 0 0 0 0 17 0 0 0 0 0 17 0 0 0 0 0 17 0 0 1 2 0;0 33 34 33 31 0 0 33 34 33 31 33 0 33 34 33 31 33 0 33 34 33 31 31 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;69948 25383 0 64481 0 2083 320 77719 53 11772 0 154784 53 202 53 387 5 8 19 35 31 95;0 0 0 33 34 34 34 34 0 33 0 33 0 0 0 1 2 0 0 0 0 0;21 21 0 55 56 56 56 56 0 55 0 21 0 33 0 1 2 0 21 22 33 34;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;222 13 29 13154 6280 149 909 2 134039 0 12656 2 3866 737 94 1690 1039 197;0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;21 22 0 21 33 33 33 0 9 10 10 0 31 32 32 0 0 33;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;5541 3833 1043664 3928 88 360 5449 0 4296 35162 179 2 342 5 0;0 0 21 0 9 0 0 0 0 0 0 0 1 2 0;33 33 27 33 13 33 33 0 33 33 34 0 1 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;12 2046 11692 2631 4920 2414 81 7271;0 0 0 0 0 0 0 0;33 34 33 33 33 33 33 34;1 1 1 1 1 1 1 1; -;4603 21529 1630 347 1 961 110 5;7 8 0 0 0 1 2 2;11 12 33 33 0 1 2 2;1 1 1 1 1 1 1 1; -;3106 3 15739 294 2 2 1765 68 2 161 398 2 8676 192 5 0;0 0 0 0 0 0 0 0 0 0 0 0 1 2 2 0;33 0 33 33 0 0 33 34 0 33 33 0 1 2 2 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;14 105 38 774 288 183001 583 1676 28 18 137 116 71 1 123 1 147 16 2 0 631 5;0 0 0 17 18 18 18 18 18 18 0 0 0 0 0 0 0 0 0 0 1 2;0 33 34 13 0 27 33 34 33 34 21 33 34 0 33 0 33 0 0 0 1 2;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; -;26324 27 4071 456 159 28 18 2 277;17 18 18 18 18 18 18 0 0;13 14 33 13 14 33 34 0 0;1 1 1 1 1 1 1 1 1; -;49 854 7016 48 5987 468 39 479 2675 1193 0 7 7 7 1 367 361 1 745 361 1 14 217 350 16;27 28 28 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0;35 36 36 36 33 34 33 34 33 0 0 0 0 0 0 33 33 0 33 33 0 0 33 34 0;1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1; diff --git a/PaddleNLP/unarchived/chinese_ner/infer.py b/PaddleNLP/unarchived/chinese_ner/infer.py deleted file mode 100644 index 02aa9fb5fb3f0f1944f9335a99485e6e4aff8754..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/infer.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import numpy as np -import argparse -import time - -import paddle.fluid as fluid -import paddle.fluid.profiler as profiler -import paddle - -import reader - - -def parse_args(): - parser = argparse.ArgumentParser("Run inference.") - parser.add_argument( - '--batch_size', - type=int, - default=6, - help='The size of a batch. (default: %(default)d)') - parser.add_argument( - '--device', - type=str, - default='GPU', - choices=['CPU', 'GPU'], - help='The device type. (default: %(default)s)') - parser.add_argument( - '--model_path', - type=str, - default='output/params_pass_0', - help='A path to the model. (default: %(default)s)') - parser.add_argument( - '--test_data_dir', - type=str, - default='data/test_files', - help='A directory with test data files. (default: %(default)s)') - parser.add_argument( - '--test_label_file', - type=str, - default='data/label_dict', - help='A file with test labels. (default: %(default)s)') - parser.add_argument( - '--num_passes', type=int, default=1, help='The number of passes.') - parser.add_argument( - '--skip_pass_num', - type=int, - default=0, - help='The first num of passes to skip in statistics calculations.') - parser.add_argument( - '--profile', action='store_true', help='If set, do profiling.') - args = parser.parse_args() - return args - - -def print_arguments(args): - print('----------- Configuration Arguments -----------') - for arg, value in sorted(vars(args).items()): - print('%s: %s' % (arg, value)) - print('------------------------------------------------') - - -def load_reverse_dict(dict_path): - return dict((idx, line.strip().split("\t")[0]) - for idx, line in enumerate(open(dict_path, "r").readlines())) - - -def to_lodtensor(data, place): - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for l in seq_lens: - cur_len += l - lod.append(cur_len) - flattened_data = np.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - res = fluid.LoDTensor() - res.set(flattened_data, place) - res.set_lod([lod]) - return res - - -def infer(args): - word = fluid.layers.data(name='word', shape=[1], dtype='int64', lod_level=1) - mention = fluid.layers.data( - name='mention', shape=[1], dtype='int64', lod_level=1) - target = fluid.layers.data( - name='target', shape=[1], dtype='int64', lod_level=1) - - label_reverse_dict = load_reverse_dict(args.test_label_file) - - test_data = paddle.batch( - reader.file_reader(args.test_data_dir), batch_size=args.batch_size) - place = fluid.CUDAPlace(0) if args.device == 'GPU' else fluid.CPUPlace() - feeder = fluid.DataFeeder(feed_list=[word, mention, target], place=place) - exe = fluid.Executor(place) - - inference_scope = fluid.Scope() - with fluid.scope_guard(inference_scope): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(args.model_path, exe) - total_passes = args.num_passes + args.skip_pass_num - batch_times = [0] * total_passes - word_counts = [0] * total_passes - wpses = [0] * total_passes - all_iters = 0 - for pass_id in range(total_passes): - if pass_id < args.skip_pass_num: - print("Warm-up pass") - if pass_id == args.skip_pass_num: - profiler.reset_profiler() - iters = 0 - for data in test_data(): - word = to_lodtensor(list(map(lambda x: x[0], data)), place) - mention = to_lodtensor(list(map(lambda x: x[1], data)), place) - - start = time.time() - crf_decode = exe.run(inference_program, - feed={"word": word, - "mention": mention}, - fetch_list=fetch_targets, - return_numpy=False) - batch_time = time.time() - start - lod_info = (crf_decode[0].lod())[0] - np_data = np.array(crf_decode[0]) - word_count = 0 - assert len(data) == len(lod_info) - 1 - for sen_index in range(len(data)): - assert len(data[sen_index][0]) == lod_info[ - sen_index + 1] - lod_info[sen_index] - word_index = 0 - for tag_index in range(lod_info[sen_index], - lod_info[sen_index + 1]): - word = str(data[sen_index][0][word_index]) - gold_tag = label_reverse_dict[data[sen_index][2][ - word_index]] - tag = label_reverse_dict[np_data[tag_index][0]] - word_index += 1 - word_count += word_index - batch_times[pass_id] += batch_time - word_counts[pass_id] += word_count - iters += 1 - all_iters += 1 - batch_times[pass_id] /= iters - word_counts[pass_id] /= iters - wps = word_counts[pass_id] / batch_times[pass_id] - wpses[pass_id] = wps - - print( - "Pass: %d, iterations (total): %d (%d), latency: %.5f s, words: %d, wps: %f" - % (pass_id, iters, all_iters, batch_times[pass_id], - word_counts[pass_id], wps)) - - # Postprocess benchmark data - latencies = batch_times[args.skip_pass_num:] - latency_avg = np.average(latencies) - latency_std = np.std(latencies) - latency_pc99 = np.percentile(latencies, 99) - wps_avg = np.average(wpses) - wps_std = np.std(wpses) - wps_pc01 = np.percentile(wpses, 1) - - # Benchmark output - print('\nTotal passes (incl. warm-up): %d' % (total_passes)) - print('Total iterations (incl. warm-up): %d' % (all_iters)) - print('Total examples (incl. warm-up): %d' % (all_iters * args.batch_size)) - print('avg latency: %.5f, std latency: %.5f, 99pc latency: %.5f' % - (latency_avg, latency_std, latency_pc99)) - print('avg wps: %.5f, std wps: %.5f, wps for 99pc latency: %.5f' % - (wps_avg, wps_std, wps_pc01)) - - -if __name__ == "__main__": - args = parse_args() - print_arguments(args) - if args.profile: - if args.device == 'GPU': - with profiler.cuda_profiler("cuda_profiler.txt", 'csv') as nvprof: - infer(args) - else: - with profiler.profiler('CPU', sorted_key='total') as cpuprof: - infer(args) - else: - infer(args) diff --git a/PaddleNLP/unarchived/chinese_ner/reader.py b/PaddleNLP/unarchived/chinese_ner/reader.py deleted file mode 100644 index c482d5e4b25086d74fafd768098b5159f8bd88b6..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/reader.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - - -def file_reader(file_dir): - def reader(): - files = os.listdir(file_dir) - for fi in files: - for line in open(file_dir + '/' + fi, 'r'): - line = line.strip() - features = line.split(";") - word_idx = [] - for item in features[1].strip().split(" "): - word_idx.append(int(item)) - target_idx = [] - for item in features[2].strip().split(" "): - label_index = int(item) - if label_index == 0: - label_index = 48 - else: - label_index -= 1 - target_idx.append(label_index) - mention_idx = [] - for item in features[3].strip().split(" "): - mention_idx.append(int(item)) - yield word_idx, mention_idx, target_idx, - - return reader diff --git a/PaddleNLP/unarchived/chinese_ner/scripts/README.md b/PaddleNLP/unarchived/chinese_ner/scripts/README.md deleted file mode 100644 index 44e91b5c772e46466dadc6f5d291f0d09e3268c6..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/scripts/README.md +++ /dev/null @@ -1,36 +0,0 @@ -## Purpose of this directory -The purpose of this directory is to provide exemplary execution commands. They are inside bash scripts described below. - -## Preparation -To add execution permissions for shell scripts, run in this directory: -`chmod +x *.sh` - -## Performance tips -Use the below environment flags for best performance: -``` -KMP_AFFINITY=granularity=fine,compact,1,0 -OMP_NUM_THREADS= -``` -For example, you can export them, or add them inside the specific files. - -## Training -### CPU with mkldnn -Run: -`./train.sh MKLDNN` -### CPU without mkldnn -Run: -`./train.sh CPU` -### GPU -Run: -`./train.sh GPU` - -## Inference -### CPU with mkldnn -Run: -`./infer.sh MKLDNN` -### CPU without mkldnn -Run: -`./infer.sh CPU` -### GPU -Run: -`./infer.sh GPU` diff --git a/PaddleNLP/unarchived/chinese_ner/scripts/infer.sh b/PaddleNLP/unarchived/chinese_ner/scripts/infer.sh deleted file mode 100644 index 7fa3367b71996e9049da05a48a3fb245538c9f1b..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/scripts/infer.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -export MKL_NUM_THREADS=1 -export OMP_NUM_THREADS=1 - -mode=$1 # gpu, cpu, mkldnn -if [ "$mode" = "CPU" ]; then - device="CPU" - model_path="cpu_model" -elif [ "$mode" = "GPU" ]; then - device="GPU" - model_path="gpu_model" -elif [ "$mode" = "MKLDNN" ]; then - device="CPU" - model_path="mkldnn_model" - export FLAGS_use_mkldnn=1 -else - echo "Invalid mode provided. Please use one of {GPU, CPU, MKLDNN}" - exit 1 -fi - -ht=`lscpu |grep "per core"|awk -F':' '{print $2}'|xargs` -if [ $ht -eq 1 ]; then # HT is OFF - if [ -z "$KMP_AFFINITY" ]; then - export KMP_AFFINITY="granularity=fine,compact,0,0" - fi - if [ -z "$OMP_DYNAMIC" ]; then - export OMP_DYNAMIC="FALSE" - fi -else # HT is ON - if [ -z "$KMP_AFFINITY" ]; then - export KMP_AFFINITY="granularity=fine,compact,1,0" - fi -fi - -python ../infer.py \ - --device $device \ - --num_passes 1 \ - --skip_pass_num 2 \ - --profile \ - --test_data_dir ../data/test_files \ - --test_label_file ../data/label_dict \ - --model_path $model_path/params_pass_0 diff --git a/PaddleNLP/unarchived/chinese_ner/scripts/train.sh b/PaddleNLP/unarchived/chinese_ner/scripts/train.sh deleted file mode 100644 index 999c1f0cd9e8723984f8141916d0a93df60a3380..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/scripts/train.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -export MKL_NUM_THREADS=1 -export OMP_NUM_THREADS=1 - -mode=$1 # gpu, cpu, mkldnn -if [ "$mode" = "CPU" ]; then - device="CPU" - parallel="--parallel True" - save_model_dir="cpu_model" -elif [ "$mode" = "GPU" ]; then - device="GPU" - parallel="--parallel True" - save_model_dir="gpu_model" -elif [ "$mode" = "MKLDNN" ]; then - device="CPU" - parallel="" - save_model_dir="mkldnn_model" - export FLAGS_use_mkldnn=1 -else - echo "Invalid mode provided. Please use one of {GPU, CPU, MKLDNN}" - exit 1 -fi - -ht=`lscpu |grep "per core"|awk -F':' '{print $2}'|xargs` -if [ $ht -eq 1 ]; then # HT is OFF - if [ -z "$KMP_AFFINITY" ]; then - export KMP_AFFINITY="granularity=fine,compact,0,0" - fi - if [ -z "$OMP_DYNAMIC" ]; then - export OMP_DYNAMIC="FALSE" - fi -else # HT is ON - if [ -z "$KMP_AFFINITY" ]; then - export KMP_AFFINITY="granularity=fine,compact,1,0" - fi -fi - -python ../train.py \ - --device $device \ - $parallel \ - --model_save_dir $save_model_dir \ - --test_data_dir ../data/test_files \ - --train_data_dir ../data/train_files \ - --num_passes 1 diff --git a/PaddleNLP/unarchived/chinese_ner/train.py b/PaddleNLP/unarchived/chinese_ner/train.py deleted file mode 100644 index 5f0ddb8cb6d87a3308c5880511a457ab9976e88f..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/chinese_ner/train.py +++ /dev/null @@ -1,415 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import math -import time -import argparse - -import numpy as np -import paddle -import paddle.fluid as fluid -from paddle.fluid.initializer import NormalInitializer - -import reader - - -def parse_args(): - parser = argparse.ArgumentParser("Run training.") - parser.add_argument( - '--batch_size', - type=int, - default=256, - help='The size of a batch. (default: %(default)d)') - parser.add_argument( - '--word_dict_len', - type=int, - default=1942563, - help='The lenght of the word dictionary. (default: %(default)d)') - parser.add_argument( - '--label_dict_len', - type=int, - default=49, - help='The lenght of the label dictionary. (default: %(default)d)') - parser.add_argument( - '--device', - type=str, - default='GPU', - choices=['CPU', 'GPU'], - help='The device type. (default: %(default)s)') - parser.add_argument( - '--train_data_dir', - type=str, - default='data/train_files', - help='A directory with train data files. (default: %(default)s)') - parser.add_argument( - '--parallel', - type=bool, - default=False, - help="Whether to use parallel training. (default: %(default)s)") - parser.add_argument( - '--test_data_dir', - type=str, - default='data/test_files', - help='A directory with test data files. (default: %(default)s)') - parser.add_argument( - '--model_save_dir', - type=str, - default='./output', - help='A directory for saving models. (default: %(default)s)') - parser.add_argument( - '--num_passes', - type=int, - default=1000, - help='The number of epochs. (default: %(default)d)') - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run the task with continuous evaluation logs.') - args = parser.parse_args() - return args - - -def print_arguments(args): - print('----------- Configuration Arguments -----------') - for arg, value in sorted(vars(args).items()): - print('%s: %s' % (arg, value)) - print('------------------------------------------------') - - -def load_reverse_dict(dict_path): - return dict((idx, line.strip().split("\t")[0]) - for idx, line in enumerate(open(dict_path, "r").readlines())) - - -def to_lodtensor(data, place): - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for l in seq_lens: - cur_len += l - lod.append(cur_len) - flattened_data = np.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - res = fluid.LoDTensor() - res.set(flattened_data, place) - res.set_lod([lod]) - return res - - -def ner_net(word_dict_len, label_dict_len): - IS_SPARSE = False - word_dim = 32 - mention_dict_len = 57 - mention_dim = 20 - grnn_hidden = 36 - emb_lr = 5 - init_bound = 0.1 - - def _net_conf(word, mark, target): - word_embedding = fluid.layers.embedding( - input=word, - size=[word_dict_len, word_dim], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr=fluid.ParamAttr( - learning_rate=emb_lr, - name="word_emb", - initializer=fluid.initializer.Uniform( - low=-init_bound, high=init_bound))) - - mention_embedding = fluid.layers.embedding( - input=mention, - size=[mention_dict_len, mention_dim], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr=fluid.ParamAttr( - learning_rate=emb_lr, - name="mention_emb", - initializer=fluid.initializer.Uniform( - low=-init_bound, high=init_bound))) - - word_embedding_r = fluid.layers.embedding( - input=word, - size=[word_dict_len, word_dim], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr=fluid.ParamAttr( - learning_rate=emb_lr, - name="word_emb_r", - initializer=fluid.initializer.Uniform( - low=-init_bound, high=init_bound))) - - mention_embedding_r = fluid.layers.embedding( - input=mention, - size=[mention_dict_len, mention_dim], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr=fluid.ParamAttr( - learning_rate=emb_lr, - name="mention_emb_r", - initializer=fluid.initializer.Uniform( - low=-init_bound, high=init_bound))) - - word_mention_vector = fluid.layers.concat( - input=[word_embedding, mention_embedding], axis=1) - - word_mention_vector_r = fluid.layers.concat( - input=[word_embedding_r, mention_embedding_r], axis=1) - - pre_gru = fluid.layers.fc( - input=word_mention_vector, - size=grnn_hidden * 3, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-init_bound, high=init_bound), - regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) - gru = fluid.layers.dynamic_gru( - input=pre_gru, - size=grnn_hidden, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-init_bound, high=init_bound), - regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) - - pre_gru_r = fluid.layers.fc( - input=word_mention_vector_r, - size=grnn_hidden * 3, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-init_bound, high=init_bound), - regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) - gru_r = fluid.layers.dynamic_gru( - input=pre_gru_r, - size=grnn_hidden, - is_reverse=True, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-init_bound, high=init_bound), - regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) - - gru_merged = fluid.layers.concat(input=[gru, gru_r], axis=1) - - emission = fluid.layers.fc( - size=label_dict_len, - input=gru_merged, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-init_bound, high=init_bound), - regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) - - crf_cost = fluid.layers.linear_chain_crf( - input=emission, - label=target, - param_attr=fluid.ParamAttr( - name='crfw', - learning_rate=0.2, )) - avg_cost = fluid.layers.mean(x=crf_cost) - return avg_cost, emission - - word = fluid.layers.data(name='word', shape=[1], dtype='int64', lod_level=1) - mention = fluid.layers.data( - name='mention', shape=[1], dtype='int64', lod_level=1) - target = fluid.layers.data( - name="target", shape=[1], dtype='int64', lod_level=1) - - avg_cost, emission = _net_conf(word, mention, target) - - return avg_cost, emission, word, mention, target - - -def test2(exe, chunk_evaluator, inference_program, test_data, place, - cur_fetch_list): - chunk_evaluator.reset() - for data in test_data(): - word = to_lodtensor(list(map(lambda x: x[0], data)), place) - mention = to_lodtensor(list(map(lambda x: x[1], data)), place) - target = to_lodtensor(list(map(lambda x: x[2], data)), place) - result_list = exe.run( - inference_program, - feed={"word": word, - "mention": mention, - "target": target}, - fetch_list=cur_fetch_list) - number_infer = np.array(result_list[0]) - number_label = np.array(result_list[1]) - number_correct = np.array(result_list[2]) - chunk_evaluator.update(number_infer[0].astype('int64'), - number_label[0].astype('int64'), - number_correct[0].astype('int64')) - return chunk_evaluator.eval() - - -def test(test_exe, chunk_evaluator, inference_program, test_data, place, - cur_fetch_list): - chunk_evaluator.reset() - for data in test_data(): - word = to_lodtensor(list(map(lambda x: x[0], data)), place) - mention = to_lodtensor(list(map(lambda x: x[1], data)), place) - target = to_lodtensor(list(map(lambda x: x[2], data)), place) - result_list = test_exe.run( - fetch_list=cur_fetch_list, - feed={"word": word, - "mention": mention, - "target": target}) - number_infer = np.array(result_list[0]) - number_label = np.array(result_list[1]) - number_correct = np.array(result_list[2]) - chunk_evaluator.update(number_infer.sum().astype('int64'), - number_label.sum().astype('int64'), - number_correct.sum().astype('int64')) - return chunk_evaluator.eval() - - -def main(args): - if not os.path.exists(args.model_save_dir): - os.makedirs(args.model_save_dir) - - main = fluid.Program() - startup = fluid.Program() - if args.enable_ce: - SEED = 102 - main.random_seed = SEED - startup.random_seed = SEED - with fluid.program_guard(main, startup): - avg_cost, feature_out, word, mention, target = ner_net( - args.word_dict_len, args.label_dict_len) - - crf_decode = fluid.layers.crf_decoding( - input=feature_out, param_attr=fluid.ParamAttr(name='crfw')) - - (precision, recall, f1_score, num_infer_chunks, num_label_chunks, - num_correct_chunks) = fluid.layers.chunk_eval( - input=crf_decode, - label=target, - chunk_scheme="IOB", - num_chunk_types=int(math.ceil((args.label_dict_len - 1) / 2.0))) - - inference_program = fluid.default_main_program().clone(for_test=True) - - sgd_optimizer = fluid.optimizer.SGD(learning_rate=1e-3) - sgd_optimizer.minimize(avg_cost) - - chunk_evaluator = fluid.metrics.ChunkEvaluator() - - train_reader = paddle.batch( - paddle.reader.shuffle( - reader.file_reader(args.train_data_dir), buf_size=2000000), - batch_size=args.batch_size) - test_reader = paddle.batch( - paddle.reader.shuffle( - reader.file_reader(args.test_data_dir), buf_size=2000000), - batch_size=args.batch_size) - - place = fluid.CUDAPlace(0) if args.device == 'GPU' else fluid.CPUPlace() - feeder = fluid.DataFeeder( - feed_list=[word, mention, target], place=place) - - exe = fluid.Executor(place) - - exe.run(startup) - if args.parallel: - train_exe = fluid.ParallelExecutor( - loss_name=avg_cost.name, use_cuda=(args.device == 'GPU')) - test_exe = fluid.ParallelExecutor( - use_cuda=(args.device == 'GPU'), - main_program=inference_program, - share_vars_from=train_exe) - else: - train_exe = exe - test_exe = exe - - total_time = 0 - ce_info = [] - batch_id = 0 - for pass_id in range(args.num_passes): - chunk_evaluator.reset() - train_reader_iter = train_reader() - start_time = time.time() - while True: - try: - cur_batch = next(train_reader_iter) - cost, nums_infer, nums_label, nums_correct = train_exe.run( - fetch_list=[ - avg_cost.name, num_infer_chunks.name, - num_label_chunks.name, num_correct_chunks.name - ], - feed=feeder.feed(cur_batch)) - chunk_evaluator.update( - np.array(nums_infer).sum().astype("int64"), - np.array(nums_label).sum().astype("int64"), - np.array(nums_correct).sum().astype("int64")) - cost_list = np.array(cost) - batch_id += 1 - except StopIteration: - break - end_time = time.time() - total_time += end_time - start_time - print("pass_id:" + str(pass_id) + ", time_cost:" + str( - end_time - start_time) + "s") - precision, recall, f1_score = chunk_evaluator.eval() - print("[Train] precision:" + str(precision) + ", recall:" + str( - recall) + ", f1:" + str(f1_score)) - ce_info.append(recall) - p, r, f1 = test2( - exe, chunk_evaluator, inference_program, test_reader, place, - [num_infer_chunks, num_label_chunks, num_correct_chunks]) - print("[Test] precision:" + str(p) + ", recall:" + str(r) + ", f1:" - + str(f1)) - save_dirname = os.path.join(args.model_save_dir, - "params_pass_%d" % pass_id) - fluid.io.save_inference_model(save_dirname, ['word', 'mention'], - [crf_decode], exe) - # only for ce - if args.enable_ce: - ce_recall = 0 - try: - ce_recall = ce_info[-2] - except: - print("ce info error") - epoch_idx = args.num_passes - device = get_device(args) - if args.device == "GPU": - gpu_num = device[1] - print("kpis\teach_pass_duration_gpu%s\t%s" % - (gpu_num, total_time / epoch_idx)) - print("kpis\ttrain_recall_gpu%s\t%s" % (gpu_num, ce_recall)) - else: - cpu_num = device[1] - threads_num = device[2] - print("kpis\teach_pass_duration_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, total_time / epoch_idx)) - print("kpis\ttrain_recall_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, ce_recall)) - - -def get_device(args): - if args.device == "GPU": - gpus = os.environ.get("CUDA_VISIBLE_DEVICES", "") - gpu_num = len(gpus.split(',')) - return "gpu", gpu_num - else: - threads_num = os.environ.get('NUM_THREADS', 1) - cpu_num = os.environ.get('CPU_NUM', 1) - return "cpu", int(cpu_num), int(threads_num) - - -if __name__ == "__main__": - args = parse_args() - print_arguments(args) - main(args) diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/.run_ce.sh b/PaddleNLP/unarchived/deep_attention_matching_net/.run_ce.sh deleted file mode 100755 index 6c1c0a344dd84b2d252b77b8d0f5de94bb0da13c..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/.run_ce.sh +++ /dev/null @@ -1,15 +0,0 @@ -###!/bin/bash -####This file is only used for continuous evaluation. - -export CE_MODE_X=1 -export CUDA_VISIBLE_DEVICES=0 -export FLAGS_eager_delete_tensor_gb=0.0 -if [ ! -e data_small.pkl ]; then - wget -c http://dam-data.bj.bcebos.com/data_small.pkl -fi - -python train_and_evaluate.py --data_path data_small.pkl \ - --use_cuda \ - --use_pyreader \ - --num_scan_data 1 \ - --batch_size 100 | python _ce.py diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/README.md b/PaddleNLP/unarchived/deep_attention_matching_net/README.md deleted file mode 100644 index 37085fe46ee6774b3e553a35d840eb11395da8a0..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# __Deep Attention Matching Network__ - -This is the source code of Deep Attention Matching network (DAM), that is proposed for multi-turn response selection in the retrieval-based chatbot. - -DAM is a neural matching network that entirely based on attention mechanism. The motivation of DAM is to capture those semantic dependencies, among dialogue elements at different level of granularities, in multi-turn conversation as matching evidences, in order to better match response candidate with its multi-turn context. DAM appears on ACL-2018, please find our paper at [http://aclweb.org/anthology/P18-1103](http://aclweb.org/anthology/P18-1103). - - -## __Network__ - -DAM is inspired by Transformer in Machine Translation (Vaswani et al., 2017), and we extend the key attention mechanism of Transformer in two perspectives and introduce those two kinds of attention in one uniform neural network. - -- **self-attention** To gradually capture semantic representations in different granularities by stacking attention from word-level embeddings. Those multi-grained semantic representations would facilitate exploring segmental dependencies between context and response. - -- **cross-attention** Attention across context and response can generally capture the relevance in dependency between segment pairs, which could provide complementary information to textual relevance for matching response with multi-turn context. - -

-
-Overview of Deep Attention Matching Network -

- -## __Results__ - -We test DAM on two large-scale multi-turn response selection tasks, i.e., the Ubuntu Corpus v1 and Douban Conversation Corpus, experimental results are bellow: - -

-
-

- -## __Usage__ - -Take the experiment on the Ubuntu Corpus v1 for Example. - -1) Go to the `ubuntu` directory - -``` -cd ubuntu -``` -2) Download the well-preprocessed data for training - -``` -sh download_data.sh -``` -3) Execute the model training and evaluation by - -``` -sh train.sh -``` -for more detailed explanation about the arguments, please run - -``` -python ../train_and_evaluate.py --help -``` - -By default, the training is executed on one single GPU, which can be switched to multiple-GPU mode easily by simply resetting the visible devices in `train.sh`, e.g., - -``` -export CUDA_VISIBLE_DEVICES=0,1,2,3 -``` - -4) Run test by - -``` -sh test.sh -``` -and run the test for different saved models by using different argument `--model_path`. - -Similary, one can carry out the experiment on the Douban Conversation Corpus by going to the directory `douban` and following the same procedure. - -## __Dependencies__ - -- Python >= 2.7.3 -- PaddlePaddle latest develop branch - -## __Citation__ - -The following article describe the DAM in detail. We recommend citing this article as default. - -``` -@inproceedings{ , - title={Multi-Turn Response Selection for Chatbots with Deep Attention Matching Network}, - author={Xiangyang Zhou, Lu Li, Daxiang Dong, Yi Liu, Ying Chen, Wayne Xin Zhao, Dianhai Yu and Hua Wu}, - booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, - volume={1}, - pages={ -- }, - year={2018} -} -``` diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/_ce.py b/PaddleNLP/unarchived/deep_attention_matching_net/_ce.py deleted file mode 100644 index 7ad30288074da3124c33fad6c96fd369a812c77c..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/_ce.py +++ /dev/null @@ -1,46 +0,0 @@ -####this file is only used for continuous evaluation test! - -import os -import sys -sys.path.append(os.environ['ceroot']) -from kpi import CostKpi, DurationKpi, AccKpi - -#### NOTE kpi.py should shared in models in some way!!!! - -train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True) -train_duration_kpi = DurationKpi('train_duration', 0.05, 0, actived=True) - -tracking_kpis = [ - train_cost_kpi, - train_duration_kpi, -] - - -def parse_log(log): - for line in log.split('\n'): - fs = line.strip().split('\t') - print(fs) - if len(fs) == 3 and fs[0] == 'kpis': - print("-----%s" % fs) - kpi_name = fs[1] - kpi_value = float(fs[2]) - yield kpi_name, kpi_value - - -def log_to_ce(log): - kpi_tracker = {} - for kpi in tracking_kpis: - kpi_tracker[kpi.name] = kpi - - for (kpi_name, kpi_value) in parse_log(log): - print(kpi_name, kpi_value) - kpi_tracker[kpi_name].add_record(kpi_value) - kpi_tracker[kpi_name].persist() - - -if __name__ == '__main__': - log = sys.stdin.read() - print("*****") - print(log) - print("****") - log_to_ce(log) diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/douban/download_data.sh b/PaddleNLP/unarchived/deep_attention_matching_net/douban/download_data.sh deleted file mode 100644 index 9f8f41719df68eae948629ccd9d38f0a42f1b746..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/douban/download_data.sh +++ /dev/null @@ -1,19 +0,0 @@ -url=http://dam-data.cdn.bcebos.com/douban.tar.gz -md5=e07ca68f21c20e09efb3e8b247194405 - -if [ ! -e douban.tar.gz ]; then - wget -c $url -fi - -echo "Checking md5 sum ..." -md5sum_tmp=`md5sum douban.tar.gz | cut -d ' ' -f1` - -if [ $md5sum_tmp != $md5 ]; then - echo "Md5sum check failed, please remove and redownload douban.tar.gz" - exit 1 -fi - -echo "Untar douban.tar.gz ..." - -tar -xzvf douban.tar.gz - diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/douban/test.sh b/PaddleNLP/unarchived/deep_attention_matching_net/douban/test.sh deleted file mode 100644 index e726124f183c52d901f1c08610ceaebcd08533cd..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/douban/test.sh +++ /dev/null @@ -1,12 +0,0 @@ -export CUDA_VISIBLE_DEVICES=0 -python -u ../test_and_evaluate.py --use_cuda \ - --ext_eval \ - --data_path ./data/data.pkl \ - --save_path ./eval_3900 \ - --model_path models/step_3900 \ - --channel1_num 16 \ - --batch_size 200 \ - --vocab_size 172130 \ - --emb_size 200 \ - --_EOS_ 1 - diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/douban/train.sh b/PaddleNLP/unarchived/deep_attention_matching_net/douban/train.sh deleted file mode 100644 index 6ed91319a7880f00a1f8b202ebe057ca1145f615..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/douban/train.sh +++ /dev/null @@ -1,14 +0,0 @@ -export CUDA_VISIBLE_DEVICES=0 -export FLAGS_eager_delete_tensor_gb=0.0 -python -u ../train_and_evaluate.py --use_cuda \ - --data_path ./data/data.pkl \ - --ext_eval \ - --word_emb_init ./data/word_embedding.pkl \ - --save_path ./models \ - --use_pyreader \ - --batch_size 256 \ - --vocab_size 172130 \ - --channel1_num 16 \ - --emb_size 200 \ - --_EOS_ 1 - diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/images/Figure1.png b/PaddleNLP/unarchived/deep_attention_matching_net/images/Figure1.png deleted file mode 100644 index c02c3e4fb1c7d5c50da4b50d19f5d182d8291f98..0000000000000000000000000000000000000000 Binary files a/PaddleNLP/unarchived/deep_attention_matching_net/images/Figure1.png and /dev/null differ diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/images/Figure2.png b/PaddleNLP/unarchived/deep_attention_matching_net/images/Figure2.png deleted file mode 100644 index b225afc1205fd03bc61301a112628447e142bb3e..0000000000000000000000000000000000000000 Binary files a/PaddleNLP/unarchived/deep_attention_matching_net/images/Figure2.png and /dev/null differ diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/model.py b/PaddleNLP/unarchived/deep_attention_matching_net/model.py deleted file mode 100644 index e7a531c71a2eec5d1f0262f20ee19f68930c467e..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/model.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import six -import numpy as np -import paddle.fluid as fluid -import utils.layers as layers - - -class Net(object): - def __init__(self, max_turn_num, max_turn_len, vocab_size, emb_size, - stack_num, channel1_num, channel2_num): - - self._max_turn_num = max_turn_num - self._max_turn_len = max_turn_len - self._vocab_size = vocab_size - self._emb_size = emb_size - self._stack_num = stack_num - self._channel1_num = channel1_num - self._channel2_num = channel2_num - self._feed_names = [] - self.word_emb_name = "shared_word_emb" - self.use_stack_op = True - self.use_mask_cache = True - self.use_sparse_embedding = True - - def create_py_reader(self, capacity, name): - # turns ids - shapes = [[-1, self._max_turn_len, 1] - for i in six.moves.xrange(self._max_turn_num)] - dtypes = ["int64" for i in six.moves.xrange(self._max_turn_num)] - # turns mask - shapes += [[-1, self._max_turn_len, 1] - for i in six.moves.xrange(self._max_turn_num)] - dtypes += ["float32" for i in six.moves.xrange(self._max_turn_num)] - - # response ids, response mask, label - shapes += [[-1, self._max_turn_len, 1], [-1, self._max_turn_len, 1], - [-1, 1]] - dtypes += ["int64", "float32", "float32"] - - py_reader = fluid.layers.py_reader( - capacity=capacity, - shapes=shapes, - lod_levels=[0] * (2 * self._max_turn_num + 3), - dtypes=dtypes, - name=name, - use_double_buffer=True) - - data_vars = fluid.layers.read_file(py_reader) - - self.turns_data = data_vars[0:self._max_turn_num] - self.turns_mask = data_vars[self._max_turn_num:2 * self._max_turn_num] - self.response = data_vars[-3] - self.response_mask = data_vars[-2] - self.label = data_vars[-1] - return py_reader - - def create_data_layers(self): - self._feed_names = [] - - self.turns_data = [] - for i in six.moves.xrange(self._max_turn_num): - name = "turn_%d" % i - turn = fluid.layers.data( - name=name, shape=[self._max_turn_len, 1], dtype="int64") - self.turns_data.append(turn) - self._feed_names.append(name) - - self.turns_mask = [] - for i in six.moves.xrange(self._max_turn_num): - name = "turn_mask_%d" % i - turn_mask = fluid.layers.data( - name=name, shape=[self._max_turn_len, 1], dtype="float32") - self.turns_mask.append(turn_mask) - self._feed_names.append(name) - - self.response = fluid.layers.data( - name="response", shape=[self._max_turn_len, 1], dtype="int64") - self.response_mask = fluid.layers.data( - name="response_mask", - shape=[self._max_turn_len, 1], - dtype="float32") - self.label = fluid.layers.data(name="label", shape=[1], dtype="float32") - self._feed_names += ["response", "response_mask", "label"] - - def get_feed_names(self): - return self._feed_names - - def set_word_embedding(self, word_emb, place): - word_emb_param = fluid.global_scope().find_var( - self.word_emb_name).get_tensor() - word_emb_param.set(word_emb, place) - - def create_network(self): - mask_cache = dict() if self.use_mask_cache else None - - response_emb = fluid.layers.embedding( - input=self.response, - size=[self._vocab_size + 1, self._emb_size], - is_sparse=self.use_sparse_embedding, - param_attr=fluid.ParamAttr( - name=self.word_emb_name, - initializer=fluid.initializer.Normal(scale=0.1))) - - # response part - Hr = response_emb - Hr_stack = [Hr] - - for index in six.moves.xrange(self._stack_num): - Hr = layers.block( - name="response_self_stack" + str(index), - query=Hr, - key=Hr, - value=Hr, - d_key=self._emb_size, - q_mask=self.response_mask, - k_mask=self.response_mask, - mask_cache=mask_cache) - Hr_stack.append(Hr) - - # context part - sim_turns = [] - for t in six.moves.xrange(self._max_turn_num): - Hu = fluid.layers.embedding( - input=self.turns_data[t], - size=[self._vocab_size + 1, self._emb_size], - is_sparse=self.use_sparse_embedding, - param_attr=fluid.ParamAttr( - name=self.word_emb_name, - initializer=fluid.initializer.Normal(scale=0.1))) - Hu_stack = [Hu] - - for index in six.moves.xrange(self._stack_num): - # share parameters - Hu = layers.block( - name="turn_self_stack" + str(index), - query=Hu, - key=Hu, - value=Hu, - d_key=self._emb_size, - q_mask=self.turns_mask[t], - k_mask=self.turns_mask[t], - mask_cache=mask_cache) - Hu_stack.append(Hu) - - # cross attention - r_a_t_stack = [] - t_a_r_stack = [] - for index in six.moves.xrange(self._stack_num + 1): - t_a_r = layers.block( - name="t_attend_r_" + str(index), - query=Hu_stack[index], - key=Hr_stack[index], - value=Hr_stack[index], - d_key=self._emb_size, - q_mask=self.turns_mask[t], - k_mask=self.response_mask, - mask_cache=mask_cache) - r_a_t = layers.block( - name="r_attend_t_" + str(index), - query=Hr_stack[index], - key=Hu_stack[index], - value=Hu_stack[index], - d_key=self._emb_size, - q_mask=self.response_mask, - k_mask=self.turns_mask[t], - mask_cache=mask_cache) - - t_a_r_stack.append(t_a_r) - r_a_t_stack.append(r_a_t) - - t_a_r_stack.extend(Hu_stack) - r_a_t_stack.extend(Hr_stack) - - if self.use_stack_op: - t_a_r = fluid.layers.stack(t_a_r_stack, axis=1) - r_a_t = fluid.layers.stack(r_a_t_stack, axis=1) - else: - for index in six.moves.xrange(len(t_a_r_stack)): - t_a_r_stack[index] = fluid.layers.unsqueeze( - input=t_a_r_stack[index], axes=[1]) - r_a_t_stack[index] = fluid.layers.unsqueeze( - input=r_a_t_stack[index], axes=[1]) - - t_a_r = fluid.layers.concat(input=t_a_r_stack, axis=1) - r_a_t = fluid.layers.concat(input=r_a_t_stack, axis=1) - - # sim shape: [batch_size, 2*(stack_num+1), max_turn_len, max_turn_len] - sim = fluid.layers.matmul( - x=t_a_r, y=r_a_t, transpose_y=True, alpha=1 / np.sqrt(200.0)) - sim_turns.append(sim) - - if self.use_stack_op: - sim = fluid.layers.stack(sim_turns, axis=2) - else: - for index in six.moves.xrange(len(sim_turns)): - sim_turns[index] = fluid.layers.unsqueeze( - input=sim_turns[index], axes=[2]) - # sim shape: [batch_size, 2*(stack_num+1), max_turn_num, max_turn_len, max_turn_len] - sim = fluid.layers.concat(input=sim_turns, axis=2) - - final_info = layers.cnn_3d(sim, self._channel1_num, self._channel2_num) - loss, logits = layers.loss(final_info, self.label) - return loss, logits diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/test_and_evaluate.py b/PaddleNLP/unarchived/deep_attention_matching_net/test_and_evaluate.py deleted file mode 100644 index 3119f1bfbf9e92c97671b799ee138f0733e77e8c..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/test_and_evaluate.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import six -import numpy as np -import time -import argparse -import multiprocessing -import paddle -import paddle.fluid as fluid -import utils.reader as reader -from utils.util import print_arguments, mkdir - -try: - import cPickle as pickle #python 2 -except ImportError as e: - import pickle #python 3 - -from model import Net - - -#yapf: disable -def parse_args(): - parser = argparse.ArgumentParser("Test for DAM.") - parser.add_argument( - '--batch_size', - type=int, - default=256, - help='Batch size for training. (default: %(default)d)') - parser.add_argument( - '--num_scan_data', - type=int, - default=2, - help='Number of pass for training. (default: %(default)d)') - parser.add_argument( - '--learning_rate', - type=float, - default=1e-3, - help='Learning rate used to train. (default: %(default)f)') - parser.add_argument( - '--data_path', - type=str, - default="data/ubuntu/data_small.pkl", - help='Path to training data. (default: %(default)s)') - parser.add_argument( - '--save_path', - type=str, - default="./", - help='Path to save score and result files. (default: %(default)s)') - parser.add_argument( - '--model_path', - type=str, - default="saved_models/step_1000", - help='Path to load well-trained models. (default: %(default)s)') - parser.add_argument( - '--use_cuda', - action='store_true', - help='If set, use cuda for training.') - parser.add_argument( - '--ext_eval', - action='store_true', - help='If set, use MAP, MRR ect for evaluation.') - parser.add_argument( - '--max_turn_num', - type=int, - default=9, - help='Maximum number of utterances in context.') - parser.add_argument( - '--max_turn_len', - type=int, - default=50, - help='Maximum length of setences in turns.') - parser.add_argument( - '--word_emb_init', - type=str, - default=None, - help='Path to the initial word embedding.') - parser.add_argument( - '--vocab_size', - type=int, - default=434512, - help='The size of vocabulary.') - parser.add_argument( - '--emb_size', - type=int, - default=200, - help='The dimension of word embedding.') - parser.add_argument( - '--_EOS_', - type=int, - default=28270, - help='The id for end of sentence in vocabulary.') - parser.add_argument( - '--stack_num', - type=int, - default=5, - help='The number of stacked attentive modules in network.') - parser.add_argument( - '--channel1_num', - type=int, - default=32, - help="The channels' number of the 1st conv3d layer's output.") - parser.add_argument( - '--channel2_num', - type=int, - default=16, - help="The channels' number of the 2nd conv3d layer's output.") - args = parser.parse_args() - return args - - -#yapf: enable - - -def test(args): - if not os.path.exists(args.save_path): - mkdir(args.save_path) - if not os.path.exists(args.model_path): - raise ValueError("Invalid model init path %s" % args.model_path) - # data data_config - data_conf = { - "batch_size": args.batch_size, - "max_turn_num": args.max_turn_num, - "max_turn_len": args.max_turn_len, - "_EOS_": args._EOS_, - } - - dam = Net(args.max_turn_num, args.max_turn_len, args.vocab_size, - args.emb_size, args.stack_num, args.channel1_num, - args.channel2_num) - dam.create_data_layers() - loss, logits = dam.create_network() - - loss.persistable = True - logits.persistable = True - - # gradient clipping - fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByValue( - max=1.0, min=-1.0)) - - test_program = fluid.default_main_program().clone(for_test=True) - optimizer = fluid.optimizer.Adam( - learning_rate=fluid.layers.exponential_decay( - learning_rate=args.learning_rate, - decay_steps=400, - decay_rate=0.9, - staircase=True)) - optimizer.minimize(loss) - - if args.use_cuda: - place = fluid.CUDAPlace(0) - dev_count = fluid.core.get_cuda_device_count() - else: - place = fluid.CPUPlace() - dev_count = multiprocessing.cpu_count() - - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - fluid.io.load_persistables(exe, args.model_path) - - test_exe = fluid.ParallelExecutor( - use_cuda=args.use_cuda, main_program=test_program) - - print("start loading data ...") - with open(args.data_path, 'rb') as f: - if six.PY2: - train_data, val_data, test_data = pickle.load(f) - else: - train_data, val_data, test_data = pickle.load(f, encoding="bytes") - print("finish loading data ...") - - if args.ext_eval: - import utils.douban_evaluation as eva - eval_metrics = ["MAP", "MRR", "P@1", "R_{10}@1", "R_{10}@2", "R_{10}@5"] - else: - import utils.evaluation as eva - eval_metrics = ["R_2@1", "R_{10}@1", "R_{10}@2", "R_{10}@5"] - - test_batches = reader.build_batches(test_data, data_conf) - - test_batch_num = len(test_batches["response"]) - - print("test batch num: %d" % test_batch_num) - - print("begin inference ...") - print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) - - score_path = os.path.join(args.save_path, 'score.txt') - score_file = open(score_path, 'w') - - for it in six.moves.xrange(test_batch_num // dev_count): - feed_list = [] - for dev in six.moves.xrange(dev_count): - index = it * dev_count + dev - batch_data = reader.make_one_batch_input(test_batches, index) - feed_dict = dict(zip(dam.get_feed_names(), batch_data)) - feed_list.append(feed_dict) - - predicts = test_exe.run(feed=feed_list, fetch_list=[logits.name]) - - scores = np.array(predicts[0]) - print("step = %d" % it) - - for dev in six.moves.xrange(dev_count): - index = it * dev_count + dev - for i in six.moves.xrange(args.batch_size): - score_file.write( - str(scores[args.batch_size * dev + i][0]) + '\t' + str( - test_batches["label"][index][i]) + '\n') - - score_file.close() - - #write evaluation result - result = eva.evaluate(score_path) - result_file_path = os.path.join(args.save_path, 'result.txt') - with open(result_file_path, 'w') as out_file: - for metric, p_at in zip(eval_metrics, result): - out_file.write(metric + ": " + str(p_at) + '\n') - print('finish test') - print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) - - -if __name__ == '__main__': - args = parse_args() - print_arguments(args) - test(args) diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/train_and_evaluate.py b/PaddleNLP/unarchived/deep_attention_matching_net/train_and_evaluate.py deleted file mode 100644 index 6a26be1655d03a62ca18a0b2c2f874b7f50f0afd..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/train_and_evaluate.py +++ /dev/null @@ -1,416 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import six -import numpy as np -import time -import argparse -import multiprocessing -import paddle -import paddle.fluid as fluid -import utils.reader as reader -from utils.util import print_arguments, mkdir - -try: - import cPickle as pickle #python 2 -except ImportError as e: - import pickle #python 3 - -from model import Net - - -#yapf: disable -def parse_args(): - parser = argparse.ArgumentParser("Training DAM.") - parser.add_argument( - '--batch_size', - type=int, - default=256, - help='Batch size for training. (default: %(default)d)') - parser.add_argument( - '--num_scan_data', - type=int, - default=2, - help='Number of pass for training. (default: %(default)d)') - parser.add_argument( - '--learning_rate', - type=float, - default=1e-3, - help='Learning rate used to train. (default: %(default)f)') - parser.add_argument( - '--data_path', - type=str, - default="data/data_small.pkl", - help='Path to training data. (default: %(default)s)') - parser.add_argument( - '--save_path', - type=str, - default="saved_models", - help='Path to save trained models. (default: %(default)s)') - parser.add_argument( - '--use_cuda', - action='store_true', - help='If set, use cuda for training.') - parser.add_argument( - '--use_pyreader', - action='store_true', - help='If set, use pyreader for reading data.') - parser.add_argument( - '--ext_eval', - action='store_true', - help='If set, use MAP, MRR ect for evaluation.') - parser.add_argument( - '--max_turn_num', - type=int, - default=9, - help='Maximum number of utterances in context.') - parser.add_argument( - '--max_turn_len', - type=int, - default=50, - help='Maximum length of setences in turns.') - parser.add_argument( - '--word_emb_init', - type=str, - default=None, - help='Path to the initial word embedding.') - parser.add_argument( - '--vocab_size', - type=int, - default=434512, - help='The size of vocabulary.') - parser.add_argument( - '--emb_size', - type=int, - default=200, - help='The dimension of word embedding.') - parser.add_argument( - '--_EOS_', - type=int, - default=28270, - help='The id for the end of sentence in vocabulary.') - parser.add_argument( - '--stack_num', - type=int, - default=5, - help='The number of stacked attentive modules in network.') - parser.add_argument( - '--channel1_num', - type=int, - default=32, - help="The channels' number of the 1st conv3d layer's output.") - parser.add_argument( - '--channel2_num', - type=int, - default=16, - help="The channels' number of the 2nd conv3d layer's output.") - args = parser.parse_args() - return args - - -#yapf: enable - - -def evaluate(score_path, result_file_path): - if args.ext_eval: - import utils.douban_evaluation as eva - else: - import utils.evaluation as eva - #write evaluation result - result = eva.evaluate(score_path) - with open(result_file_path, 'w') as out_file: - for p_at in result: - out_file.write(str(p_at) + '\n') - print('finish evaluation') - print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) - - -def test_with_feed(exe, program, feed_names, fetch_list, score_path, batches, - batch_num, dev_count): - score_file = open(score_path, 'w') - for it in six.moves.xrange(batch_num // dev_count): - feed_list = [] - for dev in six.moves.xrange(dev_count): - val_index = it * dev_count + dev - batch_data = reader.make_one_batch_input(batches, val_index) - feed_dict = dict(zip(feed_names, batch_data)) - feed_list.append(feed_dict) - - predicts = exe.run(feed=feed_list, fetch_list=fetch_list) - - scores = np.array(predicts[0]) - for dev in six.moves.xrange(dev_count): - val_index = it * dev_count + dev - for i in six.moves.xrange(args.batch_size): - score_file.write( - str(scores[args.batch_size * dev + i][0]) + '\t' + str( - batches["label"][val_index][i]) + '\n') - score_file.close() - - -def test_with_pyreader(exe, program, pyreader, fetch_list, score_path, batches, - batch_num, dev_count): - def data_provider(): - for index in six.moves.xrange(batch_num): - yield reader.make_one_batch_input(batches, index) - - score_file = open(score_path, 'w') - pyreader.decorate_tensor_provider(data_provider) - it = 0 - pyreader.start() - while True: - try: - predicts = exe.run(fetch_list=fetch_list) - - scores = np.array(predicts[0]) - for dev in six.moves.xrange(dev_count): - val_index = it * dev_count + dev - for i in six.moves.xrange(args.batch_size): - score_file.write( - str(scores[args.batch_size * dev + i][0]) + '\t' + str( - batches["label"][val_index][i]) + '\n') - it += 1 - except fluid.core.EOFException: - pyreader.reset() - break - score_file.close() - - -def train(args): - if not os.path.exists(args.save_path): - os.makedirs(args.save_path) - - # data data_config - data_conf = { - "batch_size": args.batch_size, - "max_turn_num": args.max_turn_num, - "max_turn_len": args.max_turn_len, - "_EOS_": args._EOS_, - } - - dam = Net(args.max_turn_num, args.max_turn_len, args.vocab_size, - args.emb_size, args.stack_num, args.channel1_num, - args.channel2_num) - - train_program = fluid.Program() - train_startup = fluid.Program() - if "CE_MODE_X" in os.environ: - train_program.random_seed = 110 - train_startup.random_seed = 110 - with fluid.program_guard(train_program, train_startup): - with fluid.unique_name.guard(): - if args.use_pyreader: - train_pyreader = dam.create_py_reader( - capacity=10, name='train_reader') - else: - dam.create_data_layers() - loss, logits = dam.create_network() - loss.persistable = True - logits.persistable = True - # gradient clipping - fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByValue( - max=1.0, min=-1.0)) - - optimizer = fluid.optimizer.Adam( - learning_rate=fluid.layers.exponential_decay( - learning_rate=args.learning_rate, - decay_steps=400, - decay_rate=0.9, - staircase=True)) - optimizer.minimize(loss) - - test_program = fluid.Program() - test_startup = fluid.Program() - if "CE_MODE_X" in os.environ: - test_program.random_seed = 110 - test_startup.random_seed = 110 - with fluid.program_guard(test_program, test_startup): - with fluid.unique_name.guard(): - if args.use_pyreader: - test_pyreader = dam.create_py_reader( - capacity=10, name='test_reader') - else: - dam.create_data_layers() - - loss, logits = dam.create_network() - loss.persistable = True - logits.persistable = True - - test_program = test_program.clone(for_test=True) - - if args.use_cuda: - place = fluid.CUDAPlace(0) - dev_count = fluid.core.get_cuda_device_count() - else: - place = fluid.CPUPlace() - dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - - print("device count %d" % dev_count) - print("theoretical memory usage: ") - print(fluid.contrib.memory_usage( - program=train_program, batch_size=args.batch_size)) - - exe = fluid.Executor(place) - exe.run(train_startup) - exe.run(test_startup) - - train_exe = fluid.ParallelExecutor( - use_cuda=args.use_cuda, loss_name=loss.name, main_program=train_program) - - test_exe = fluid.ParallelExecutor( - use_cuda=args.use_cuda, - main_program=test_program, - share_vars_from=train_exe) - - if args.word_emb_init is not None: - print("start loading word embedding init ...") - if six.PY2: - word_emb = np.array(pickle.load(open(args.word_emb_init, - 'rb'))).astype('float32') - else: - word_emb = np.array( - pickle.load( - open(args.word_emb_init, 'rb'), encoding="bytes")).astype( - 'float32') - dam.set_word_embedding(word_emb, place) - print("finish init word embedding ...") - - print("start loading data ...") - with open(args.data_path, 'rb') as f: - if six.PY2: - train_data, val_data, test_data = pickle.load(f) - else: - train_data, val_data, test_data = pickle.load(f, encoding="bytes") - print("finish loading data ...") - - val_batches = reader.build_batches(val_data, data_conf) - - batch_num = len(train_data[six.b('y')]) // args.batch_size - val_batch_num = len(val_batches["response"]) - - print_step = max(1, batch_num // (dev_count * 100)) - save_step = max(1, batch_num // (dev_count * 10)) - - print("begin model training ...") - print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) - - # train on one epoch data by feeding - def train_with_feed(step): - ave_cost = 0.0 - for it in six.moves.xrange(batch_num // dev_count): - feed_list = [] - for dev in six.moves.xrange(dev_count): - index = it * dev_count + dev - batch_data = reader.make_one_batch_input(train_batches, index) - feed_dict = dict(zip(dam.get_feed_names(), batch_data)) - feed_list.append(feed_dict) - - cost = train_exe.run(feed=feed_list, fetch_list=[loss.name]) - - ave_cost += np.array(cost[0]).mean() - step = step + 1 - if step % print_step == 0: - print("processed: [" + str(step * dev_count * 1.0 / batch_num) + - "] ave loss: [" + str(ave_cost / print_step) + "]") - ave_cost = 0.0 - - if (args.save_path is not None) and (step % save_step == 0): - save_path = os.path.join(args.save_path, "step_" + str(step)) - print("Save model at step %d ... " % step) - print(time.strftime('%Y-%m-%d %H:%M:%S', - time.localtime(time.time()))) - fluid.io.save_persistables(exe, save_path, train_program) - - score_path = os.path.join(args.save_path, 'score.' + str(step)) - test_with_feed(test_exe, test_program, - dam.get_feed_names(), [logits.name], score_path, - val_batches, val_batch_num, dev_count) - - result_file_path = os.path.join(args.save_path, - 'result.' + str(step)) - evaluate(score_path, result_file_path) - return step, np.array(cost[0]).mean() - - # train on one epoch with pyreader - def train_with_pyreader(step): - def data_provider(): - for index in six.moves.xrange(batch_num): - yield reader.make_one_batch_input(train_batches, index) - - train_pyreader.decorate_tensor_provider(data_provider) - - ave_cost = 0.0 - train_pyreader.start() - while True: - try: - cost = train_exe.run(fetch_list=[loss.name]) - - ave_cost += np.array(cost[0]).mean() - step = step + 1 - if step % print_step == 0: - print("processed: [" + str(step * dev_count * 1.0 / - batch_num) + "] ave loss: [" + - str(ave_cost / print_step) + "]") - ave_cost = 0.0 - - if (args.save_path is not None) and (step % save_step == 0): - save_path = os.path.join(args.save_path, - "step_" + str(step)) - print("Save model at step %d ... " % step) - print(time.strftime('%Y-%m-%d %H:%M:%S', - time.localtime(time.time()))) - fluid.io.save_persistables(exe, save_path, train_program) - - score_path = os.path.join(args.save_path, - 'score.' + str(step)) - test_with_pyreader(test_exe, test_program, test_pyreader, - [logits.name], score_path, val_batches, - val_batch_num, dev_count) - - result_file_path = os.path.join(args.save_path, - 'result.' + str(step)) - evaluate(score_path, result_file_path) - - except fluid.core.EOFException: - train_pyreader.reset() - break - return step, np.array(cost[0]).mean() - - # train over different epoches - global_step, train_time = 0, 0.0 - for epoch in six.moves.xrange(args.num_scan_data): - shuffle_train = reader.unison_shuffle( - train_data, seed=110 if ("CE_MODE_X" in os.environ) else None) - train_batches = reader.build_batches(shuffle_train, data_conf) - - begin_time = time.time() - if args.use_pyreader: - global_step, last_cost = train_with_pyreader(global_step) - else: - global_step, last_cost = train_with_feed(global_step) - - pass_time_cost = time.time() - begin_time - train_time += pass_time_cost - print("Pass {0}, pass_time_cost {1}" - .format(epoch, "%2.2f sec" % pass_time_cost)) - # For internal continuous evaluation - if "CE_MODE_X" in os.environ: - print("kpis train_cost %f" % last_cost) - print("kpis train_duration %f" % train_time) - - -if __name__ == '__main__': - args = parse_args() - print_arguments(args) - train(args) diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/ubuntu/download_data.sh b/PaddleNLP/unarchived/deep_attention_matching_net/ubuntu/download_data.sh deleted file mode 100644 index 47ff08fe259e78e620fb0d461c651a086d54536e..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/ubuntu/download_data.sh +++ /dev/null @@ -1,18 +0,0 @@ -url=http://dam-data.cdn.bcebos.com/ubuntu.tar.gz -md5=9d7db116a040530a16f68dc0ab44e4b6 - -if [ ! -e ubuntu.tar.gz ]; then - wget -c $url -fi - -echo "Checking md5 sum ..." -md5sum_tmp=`md5sum ubuntu.tar.gz | cut -d ' ' -f1` - -if [ $md5sum_tmp != $md5 ]; then - echo "Md5sum check failed, please remove and redownload ubuntu.tar.gz" - exit 1 -fi - -echo "Untar ubuntu.tar.gz ..." - -tar -xzvf ubuntu.tar.gz diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/ubuntu/test.sh b/PaddleNLP/unarchived/deep_attention_matching_net/ubuntu/test.sh deleted file mode 100644 index db1657e62c6bca1bd5b42bc53be95b6f8b2391c4..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/ubuntu/test.sh +++ /dev/null @@ -1,10 +0,0 @@ -export CUDA_VISIBLE_DEVICES=0 -python -u ../test_and_evaluate.py --use_cuda \ - --data_path ./data/data.pkl \ - --save_path ./step_3900 \ - --model_path ./models/step_3900 \ - --batch_size 200 \ - --vocab_size 434512 \ - --emb_size 200 \ - --_EOS_ 28270 - diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/ubuntu/train.sh b/PaddleNLP/unarchived/deep_attention_matching_net/ubuntu/train.sh deleted file mode 100644 index 66ebc2e62f66276f808ac36dfad31470950fd9b9..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/ubuntu/train.sh +++ /dev/null @@ -1,12 +0,0 @@ -export CUDA_VISIBLE_DEVICES=0 -export FLAGS_eager_delete_tensor_gb=0.0 -python -u ../train_and_evaluate.py --use_cuda \ - --data_path ./data/data.pkl \ - --word_emb_init ./data/word_embedding.pkl \ - --save_path ./models \ - --use_pyreader \ - --batch_size 256 \ - --vocab_size 434512 \ - --emb_size 200 \ - --_EOS_ 28270 - diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/utils/__init__.py b/PaddleNLP/unarchived/deep_attention_matching_net/utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/utils/douban_evaluation.py b/PaddleNLP/unarchived/deep_attention_matching_net/utils/douban_evaluation.py deleted file mode 100644 index b94c643b363d5085a933e2e49a7de607bdfb0d88..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/utils/douban_evaluation.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import six -import numpy as np -from sklearn.metrics import average_precision_score - - -def mean_average_precision(sort_data): - #to do - count_1 = 0 - sum_precision = 0 - for index in six.moves.xrange(len(sort_data)): - if sort_data[index][1] == 1: - count_1 += 1 - sum_precision += 1.0 * count_1 / (index + 1) - return sum_precision / count_1 - - -def mean_reciprocal_rank(sort_data): - sort_lable = [s_d[1] for s_d in sort_data] - assert 1 in sort_lable - return 1.0 / (1 + sort_lable.index(1)) - - -def precision_at_position_1(sort_data): - if sort_data[0][1] == 1: - return 1 - else: - return 0 - - -def recall_at_position_k_in_10(sort_data, k): - sort_lable = [s_d[1] for s_d in sort_data] - select_lable = sort_lable[:k] - return 1.0 * select_lable.count(1) / sort_lable.count(1) - - -def evaluation_one_session(data): - sort_data = sorted(data, key=lambda x: x[0], reverse=True) - m_a_p = mean_average_precision(sort_data) - m_r_r = mean_reciprocal_rank(sort_data) - p_1 = precision_at_position_1(sort_data) - r_1 = recall_at_position_k_in_10(sort_data, 1) - r_2 = recall_at_position_k_in_10(sort_data, 2) - r_5 = recall_at_position_k_in_10(sort_data, 5) - return m_a_p, m_r_r, p_1, r_1, r_2, r_5 - - -def evaluate(file_path): - sum_m_a_p = 0 - sum_m_r_r = 0 - sum_p_1 = 0 - sum_r_1 = 0 - sum_r_2 = 0 - sum_r_5 = 0 - i = 0 - total_num = 0 - with open(file_path, 'r') as infile: - for line in infile: - if i % 10 == 0: - data = [] - - tokens = line.strip().split('\t') - data.append((float(tokens[0]), int(tokens[1]))) - if i % 10 == 9: - total_num += 1 - m_a_p, m_r_r, p_1, r_1, r_2, r_5 = evaluation_one_session(data) - sum_m_a_p += m_a_p - sum_m_r_r += m_r_r - sum_p_1 += p_1 - sum_r_1 += r_1 - sum_r_2 += r_2 - sum_r_5 += r_5 - i += 1 - #print('total num: %s' %total_num) - #print('MAP: %s' %(1.0*sum_m_a_p/total_num)) - #print('MRR: %s' %(1.0*sum_m_r_r/total_num)) - #print('P@1: %s' %(1.0*sum_p_1/total_num)) - return (1.0 * sum_m_a_p / total_num, 1.0 * sum_m_r_r / total_num, - 1.0 * sum_p_1 / total_num, 1.0 * sum_r_1 / total_num, - 1.0 * sum_r_2 / total_num, 1.0 * sum_r_5 / total_num) - - -if __name__ == '__main__': - result = evaluate(sys.argv[1]) - for r in result: - print(r) diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/utils/evaluation.py b/PaddleNLP/unarchived/deep_attention_matching_net/utils/evaluation.py deleted file mode 100644 index 49dc98ad06a8bd1f3cd0280050c0af7225ba13d4..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/utils/evaluation.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import six - - -def get_p_at_n_in_m(data, n, m, ind): - pos_score = data[ind][0] - curr = data[ind:ind + m] - curr = sorted(curr, key=lambda x: x[0], reverse=True) - - if curr[n - 1][0] <= pos_score: - return 1 - return 0 - - -def evaluate(file_path): - data = [] - with open(file_path, 'r') as file: - for line in file: - line = line.strip() - tokens = line.split("\t") - - if len(tokens) != 2: - continue - - data.append((float(tokens[0]), int(tokens[1]))) - - #assert len(data) % 10 == 0 - - p_at_1_in_2 = 0.0 - p_at_1_in_10 = 0.0 - p_at_2_in_10 = 0.0 - p_at_5_in_10 = 0.0 - - length = len(data) // 10 - - for i in six.moves.xrange(0, length): - ind = i * 10 - assert data[ind][1] == 1 - - p_at_1_in_2 += get_p_at_n_in_m(data, 1, 2, ind) - p_at_1_in_10 += get_p_at_n_in_m(data, 1, 10, ind) - p_at_2_in_10 += get_p_at_n_in_m(data, 2, 10, ind) - p_at_5_in_10 += get_p_at_n_in_m(data, 5, 10, ind) - - return (p_at_1_in_2 / length, p_at_1_in_10 / length, p_at_2_in_10 / length, - p_at_5_in_10 / length) diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/utils/layers.py b/PaddleNLP/unarchived/deep_attention_matching_net/utils/layers.py deleted file mode 100644 index cc3f12b8cd56886764eee01f2e9c65777c175e53..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/utils/layers.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import paddle.fluid as fluid - - -def loss(x, y, clip_value=10.0): - """Calculate the sigmoid cross entropy with logits for input(x). - - Args: - x: Variable with shape with shape [batch, dim] - y: Input label - - Returns: - loss: cross entropy - logits: prediction - """ - - logits = fluid.layers.fc( - input=x, - size=1, - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(0.))) - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logits, label=y) - loss = fluid.layers.reduce_mean( - fluid.layers.clip( - loss, min=-clip_value, max=clip_value)) - return loss, logits - - -def ffn(input, d_inner_hid, d_hid, name=None): - """Position-wise Feed-Forward Network - """ - - hidden = fluid.layers.fc(input=input, - size=d_inner_hid, - num_flatten_dims=2, - param_attr=fluid.ParamAttr(name=name + '_fc.w_0'), - bias_attr=fluid.ParamAttr( - name=name + '_fc.b_0', - initializer=fluid.initializer.Constant(0.)), - act="relu") - out = fluid.layers.fc(input=hidden, - size=d_hid, - num_flatten_dims=2, - param_attr=fluid.ParamAttr(name=name + '_fc.w_1'), - bias_attr=fluid.ParamAttr( - name=name + '_fc.b_1', - initializer=fluid.initializer.Constant(0.))) - return out - - -def dot_product_attention(query, - key, - value, - d_key, - q_mask=None, - k_mask=None, - dropout_rate=None, - mask_cache=None): - """Dot product layer. - - Args: - query: a tensor with shape [batch, Q_time, Q_dimension] - key: a tensor with shape [batch, time, K_dimension] - value: a tensor with shape [batch, time, V_dimension] - - q_lengths: a tensor with shape [batch] - k_lengths: a tensor with shape [batch] - - Returns: - a tensor with shape [batch, query_time, value_dimension] - - Raises: - AssertionError: if Q_dimension not equal to K_dimension when attention - type is dot. - """ - - logits = fluid.layers.matmul( - x=query, y=key, transpose_y=True, alpha=d_key**(-0.5)) - - if (q_mask is not None) and (k_mask is not None): - if mask_cache is not None and q_mask.name in mask_cache and k_mask.name in mask_cache[ - q_mask.name]: - mask, another_mask = mask_cache[q_mask.name][k_mask.name] - else: - mask = fluid.layers.matmul(x=q_mask, y=k_mask, transpose_y=True) - another_mask = fluid.layers.scale( - mask, - scale=float(2**32 - 1), - bias=float(-1), - bias_after_scale=False) - if mask_cache is not None: - if q_mask.name not in mask_cache: - mask_cache[q_mask.name] = dict() - - mask_cache[q_mask.name][k_mask.name] = [mask, another_mask] - - logits = mask * logits + another_mask - - attention = fluid.layers.softmax(logits) - if dropout_rate: - attention = fluid.layers.dropout( - input=attention, dropout_prob=dropout_rate, is_test=False, seed=2) - - atten_out = fluid.layers.matmul(x=attention, y=value) - - return atten_out - - -def block(name, - query, - key, - value, - d_key, - q_mask=None, - k_mask=None, - is_layer_norm=True, - dropout_rate=None, - mask_cache=None): - """ - """ - - att_out = dot_product_attention( - query, - key, - value, - d_key, - q_mask, - k_mask, - dropout_rate, - mask_cache=mask_cache) - - y = query + att_out - if is_layer_norm: - y = fluid.layers.layer_norm( - input=y, - begin_norm_axis=len(y.shape) - 1, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.), - name=name + '_layer_norm.w_0'), - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.), - name=name + '_layer_norm.b_0')) - - z = ffn(y, d_key, d_key, name) - w = y + z - if is_layer_norm: - w = fluid.layers.layer_norm( - input=w, - begin_norm_axis=len(w.shape) - 1, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.), - name=name + '_layer_norm.w_1'), - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.), - name=name + '_layer_norm.b_1')) - - return w - - -def cnn_3d(input, out_channels_0, out_channels_1, add_relu=True): - # same padding - conv_0 = fluid.layers.conv3d( - name="conv3d_0", - input=input, - num_filters=out_channels_0, - filter_size=[3, 3, 3], - padding=[1, 1, 1], - act="elu" if add_relu else None, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform( - low=-0.01, high=0.01)), - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.0))) - - # same padding - pooling_0 = fluid.layers.pool3d( - input=conv_0, - pool_type="max", - pool_size=3, - pool_padding=1, - pool_stride=3) - - conv_1 = fluid.layers.conv3d( - name="conv3d_1", - input=pooling_0, - num_filters=out_channels_1, - filter_size=[3, 3, 3], - padding=[1, 1, 1], - act="elu" if add_relu else None, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform( - low=-0.01, high=0.01)), - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.0))) - - # same padding - pooling_1 = fluid.layers.pool3d( - input=conv_1, - pool_type="max", - pool_size=3, - pool_padding=1, - pool_stride=3) - - return pooling_1 diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/utils/reader.py b/PaddleNLP/unarchived/deep_attention_matching_net/utils/reader.py deleted file mode 100644 index f89a7c0999e8e1cbd716a3ae7e7b721dbbdaf5a7..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/utils/reader.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import six -import numpy as np - -try: - import cPickle as pickle #python 2 -except ImportError as e: - import pickle #python 3 - - -def unison_shuffle(data, seed=None): - if seed is not None: - np.random.seed(seed) - - y = np.array(data[six.b('y')]) - c = np.array(data[six.b('c')]) - r = np.array(data[six.b('r')]) - - assert len(y) == len(c) == len(r) - p = np.random.permutation(len(y)) - print(p) - shuffle_data = {six.b('y'): y[p], six.b('c'): c[p], six.b('r'): r[p]} - return shuffle_data - - -def split_c(c, split_id): - '''c is a list, example context - split_id is a integer, conf[_EOS_] - return nested list - ''' - turns = [[]] - for _id in c: - if _id != split_id: - turns[-1].append(_id) - else: - turns.append([]) - if turns[-1] == [] and len(turns) > 1: - turns.pop() - return turns - - -def normalize_length(_list, length, cut_type='tail'): - '''_list is a list or nested list, example turns/r/single turn c - cut_type is head or tail, if _list len > length is used - return a list len=length and min(read_length, length) - ''' - real_length = len(_list) - if real_length == 0: - return [0] * length, 0 - - if real_length <= length: - if not isinstance(_list[0], list): - _list.extend([0] * (length - real_length)) - else: - _list.extend([[]] * (length - real_length)) - return _list, real_length - - if cut_type == 'head': - return _list[:length], length - if cut_type == 'tail': - return _list[-length:], length - - -def produce_one_sample(data, - index, - split_id, - max_turn_num, - max_turn_len, - turn_cut_type='tail', - term_cut_type='tail'): - '''max_turn_num=10 - max_turn_len=50 - return y, nor_turns_nor_c, nor_r, turn_len, term_len, r_len - ''' - c = data[six.b('c')][index] - r = data[six.b('r')][index][:] - y = data[six.b('y')][index] - - turns = split_c(c, split_id) - #normalize turns_c length, nor_turns length is max_turn_num - nor_turns, turn_len = normalize_length(turns, max_turn_num, turn_cut_type) - - nor_turns_nor_c = [] - term_len = [] - #nor_turn_nor_c length is max_turn_num, element is a list length is max_turn_len - for c in nor_turns: - #nor_c length is max_turn_len - nor_c, nor_c_len = normalize_length(c, max_turn_len, term_cut_type) - nor_turns_nor_c.append(nor_c) - term_len.append(nor_c_len) - - nor_r, r_len = normalize_length(r, max_turn_len, term_cut_type) - - return y, nor_turns_nor_c, nor_r, turn_len, term_len, r_len - - -def build_one_batch(data, - batch_index, - conf, - turn_cut_type='tail', - term_cut_type='tail'): - _turns = [] - _tt_turns_len = [] - _every_turn_len = [] - - _response = [] - _response_len = [] - - _label = [] - - for i in six.moves.xrange(conf['batch_size']): - index = batch_index * conf['batch_size'] + i - y, nor_turns_nor_c, nor_r, turn_len, term_len, r_len = produce_one_sample( - data, index, conf['_EOS_'], conf['max_turn_num'], - conf['max_turn_len'], turn_cut_type, term_cut_type) - - _label.append(y) - _turns.append(nor_turns_nor_c) - _response.append(nor_r) - _every_turn_len.append(term_len) - _tt_turns_len.append(turn_len) - _response_len.append(r_len) - - return _turns, _tt_turns_len, _every_turn_len, _response, _response_len, _label - - -def build_one_batch_dict(data, - batch_index, - conf, - turn_cut_type='tail', - term_cut_type='tail'): - _turns, _tt_turns_len, _every_turn_len, _response, _response_len, _label = build_one_batch( - data, batch_index, conf, turn_cut_type, term_cut_type) - ans = { - 'turns': _turns, - 'tt_turns_len': _tt_turns_len, - 'every_turn_len': _every_turn_len, - 'response': _response, - 'response_len': _response_len, - 'label': _label - } - return ans - - -def build_batches(data, conf, turn_cut_type='tail', term_cut_type='tail'): - _turns_batches = [] - _tt_turns_len_batches = [] - _every_turn_len_batches = [] - - _response_batches = [] - _response_len_batches = [] - - _label_batches = [] - - batch_len = len(data[six.b('y')]) // conf['batch_size'] - for batch_index in six.moves.range(batch_len): - _turns, _tt_turns_len, _every_turn_len, _response, _response_len, _label = build_one_batch( - data, batch_index, conf, turn_cut_type='tail', term_cut_type='tail') - - _turns_batches.append(_turns) - _tt_turns_len_batches.append(_tt_turns_len) - _every_turn_len_batches.append(_every_turn_len) - - _response_batches.append(_response) - _response_len_batches.append(_response_len) - - _label_batches.append(_label) - - ans = { - "turns": _turns_batches, - "tt_turns_len": _tt_turns_len_batches, - "every_turn_len": _every_turn_len_batches, - "response": _response_batches, - "response_len": _response_len_batches, - "label": _label_batches - } - - return ans - - -def make_one_batch_input(data_batches, index): - """Split turns and return feeding data. - - Args: - data_batches: All data batches - index: The index for current batch - - Return: - feeding dictionary - """ - - turns = np.array(data_batches["turns"][index]).astype('int64') - tt_turns_len = np.array(data_batches["tt_turns_len"][index]).astype('int64') - every_turn_len = np.array(data_batches["every_turn_len"][index]).astype( - 'int64') - response = np.array(data_batches["response"][index]).astype('int64') - response_len = np.array(data_batches["response_len"][index]).astype('int64') - - batch_size = turns.shape[0] - max_turn_num = turns.shape[1] - max_turn_len = turns.shape[2] - - turns_list = [turns[:, i, :] for i in six.moves.xrange(max_turn_num)] - every_turn_len_list = [ - every_turn_len[:, i] for i in six.moves.xrange(max_turn_num) - ] - - feed_list = [] - for i, turn in enumerate(turns_list): - turn = np.expand_dims(turn, axis=-1) - feed_list.append(turn) - - for i, turn_len in enumerate(every_turn_len_list): - turn_mask = np.ones((batch_size, max_turn_len, 1)).astype("float32") - for row in six.moves.xrange(batch_size): - turn_mask[row, turn_len[row]:, 0] = 0 - feed_list.append(turn_mask) - - response = np.expand_dims(response, axis=-1) - feed_list.append(response) - - response_mask = np.ones((batch_size, max_turn_len, 1)).astype("float32") - for row in six.moves.xrange(batch_size): - response_mask[row, response_len[row]:, 0] = 0 - feed_list.append(response_mask) - - label = np.array([data_batches["label"][index]]).reshape( - [-1, 1]).astype("float32") - feed_list.append(label) - - return feed_list - - -if __name__ == '__main__': - conf = { - "batch_size": 256, - "max_turn_num": 10, - "max_turn_len": 50, - "_EOS_": 28270, - } - with open('../ubuntu/data/data_small.pkl', 'rb') as f: - if six.PY2: - train, val, test = pickle.load(f) - else: - train, val, test = pickle.load(f, encoding="bytes") - print('load data success') - - train_batches = build_batches(train, conf) - val_batches = build_batches(val, conf) - test_batches = build_batches(test, conf) - print('build batches success') diff --git a/PaddleNLP/unarchived/deep_attention_matching_net/utils/util.py b/PaddleNLP/unarchived/deep_attention_matching_net/utils/util.py deleted file mode 100644 index f521a6f5f8cc0e12f3d7f539a2c289b63aea5b71..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/deep_attention_matching_net/utils/util.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import six -import os - - -def print_arguments(args): - print('----------- Configuration Arguments -----------') - for arg, value in sorted(six.iteritems(vars(args))): - print('%s: %s' % (arg, value)) - print('------------------------------------------------') - - -def mkdir(path): - if not os.path.isdir(path): - mkdir(os.path.split(path)[0]) - else: - return - os.mkdir(path) - - -def pos_encoding_init(): - pass - - -def scaled_dot_product_attention(): - pass diff --git a/PaddleNLP/unarchived/language_model/gru/.run_ce.sh b/PaddleNLP/unarchived/language_model/gru/.run_ce.sh deleted file mode 100644 index 5ee2d8aa0582b2b8504f9ba645b6252aa75f23bf..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/language_model/gru/.run_ce.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -export MKL_NUM_THREADS=1 -export OMP_NUM_THREADS=1 - -cudaid=${language_model:=0} # use 0-th card as default -export CUDA_VISIBLE_DEVICES=$cudaid - -FLAGS_benchmark=true python train.py --enable_ce | python _ce.py - -cudaid=${language_model_m:=0,1,2,3} # use 0,1,2,3 card as default -export CUDA_VISIBLE_DEVICES=$cudaid - -FLAGS_benchmark=true python train.py --enable_ce | python _ce.py diff --git a/PaddleNLP/unarchived/language_model/gru/README.md b/PaddleNLP/unarchived/language_model/gru/README.md deleted file mode 100644 index 91ce2d7f58085b56da2ac2dec03af2a05985ab8f..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/language_model/gru/README.md +++ /dev/null @@ -1,148 +0,0 @@ -# 语言模型 - -以下是本例的简要目录结构及说明： - -```text -. -├── README.md # 文档 -├── train.py # 训练脚本 -├── infer.py # 预测脚本 -└── utils.py # 通用函数 -``` - - -## 简介 - -循环神经网络语言模型的介绍可以参阅论文[Recurrent Neural Network Regularization](https://arxiv.org/abs/1409.2329)，在本例中，我们实现了GRU-RNN语言模型。 - -## 训练 - -运行命令 `python train.py` 开始训练模型。 -```python -python train.py -``` - -当前支持的参数可参见[train.py](./train.py) `train_net` 函数 -```python -vocab, train_reader, test_reader = utils.prepare_data( - batch_size=20, # batch size - buffer_size=1000, # buffer size, default value is OK - word_freq_threshold=0) # vocabulary related parameter, and words with frequency below this value will be filtered - -train(train_reader=train_reader, - vocab=vocab, - network=network, - hid_size=200, # embedding and hidden size - base_lr=1.0, # base learning rate - batch_size=20, # batch size, the same as that in prepare_data - pass_num=12, # the number of passes for training - use_cuda=True, # whether to use GPU card - parallel=False, # whether to be parallel - model_dir="model", # directory to save model - init_low_bound=-0.1, # uniform parameter initialization lower bound - init_high_bound=0.1) # uniform parameter initialization upper bound -``` - -## 自定义网络结构 - -可在[train.py](./train.py) `network` 函数中调整网络结构，当前的网络结构如下： -```python -emb = fluid.layers.embedding(input=src, size=[vocab_size, hid_size], - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=init_low_bound, high=init_high_bound), - learning_rate=emb_lr_x), - is_sparse=True) - -fc0 = fluid.layers.fc(input=emb, size=hid_size * 3, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=init_low_bound, high=init_high_bound), - learning_rate=gru_lr_x)) -gru_h0 = fluid.layers.dynamic_gru(input=fc0, size=hid_size, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=init_low_bound, high=init_high_bound), - learning_rate=gru_lr_x)) - -fc = fluid.layers.fc(input=gru_h0, size=vocab_size, act='softmax', - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=init_low_bound, high=init_high_bound), - learning_rate=fc_lr_x)) - -cost = fluid.layers.cross_entropy(input=fc, label=dst) -``` - -## 训练结果示例 - -我们在Tesla K40m单GPU卡上训练的日志如下所示 -```text -epoch_1 start -step:100 ppl:771.053 -step:200 ppl:449.597 -step:300 ppl:642.654 -step:400 ppl:458.128 -step:500 ppl:510.912 -step:600 ppl:451.545 -step:700 ppl:364.404 -step:800 ppl:324.272 -step:900 ppl:360.797 -step:1000 ppl:275.761 -step:1100 ppl:294.599 -step:1200 ppl:335.877 -step:1300 ppl:185.262 -step:1400 ppl:241.744 -step:1500 ppl:211.507 -step:1600 ppl:233.431 -step:1700 ppl:298.767 -step:1800 ppl:203.403 -step:1900 ppl:158.828 -step:2000 ppl:171.148 -step:2100 ppl:280.884 -epoch:1 num_steps:2104 time_cost(s):47.478780 -model saved in model/epoch_1 -epoch_2 start -step:100 ppl:238.099 -step:200 ppl:136.527 -step:300 ppl:204.184 -step:400 ppl:252.886 -step:500 ppl:177.377 -step:600 ppl:197.688 -step:700 ppl:131.650 -step:800 ppl:223.906 -step:900 ppl:144.785 -step:1000 ppl:176.286 -step:1100 ppl:148.158 -step:1200 ppl:203.581 -step:1300 ppl:168.208 -step:1400 ppl:159.412 -step:1500 ppl:114.032 -step:1600 ppl:157.985 -step:1700 ppl:147.743 -step:1800 ppl:88.676 -step:1900 ppl:141.962 -step:2000 ppl:106.087 -step:2100 ppl:122.709 -epoch:2 num_steps:2104 time_cost(s):47.583789 -model saved in model/epoch_2 -... -``` - -## 预测 -运行命令 `python infer.py model_dir start_epoch last_epoch(inclusive)` 开始预测，其中，start_epoch指定开始预测的轮次，last_epoch指定结束的轮次，例如 -```python -python infer.py model 1 12 # prediction from epoch 1 to epoch 12 -``` - -## 预测结果示例 -```text -model:model/epoch_1 ppl:254.540 time_cost(s):3.29 -model:model/epoch_2 ppl:177.671 time_cost(s):3.27 -model:model/epoch_3 ppl:156.251 time_cost(s):3.27 -model:model/epoch_4 ppl:139.036 time_cost(s):3.27 -model:model/epoch_5 ppl:132.661 time_cost(s):3.27 -model:model/epoch_6 ppl:130.092 time_cost(s):3.28 -model:model/epoch_7 ppl:128.751 time_cost(s):3.27 -model:model/epoch_8 ppl:125.411 time_cost(s):3.27 -model:model/epoch_9 ppl:124.604 time_cost(s):3.28 -model:model/epoch_10 ppl:124.754 time_cost(s):3.29 -model:model/epoch_11 ppl:125.421 time_cost(s):3.27 -model:model/epoch_12 ppl:125.676 time_cost(s):3.27 -``` diff --git a/PaddleNLP/unarchived/language_model/gru/_ce.py b/PaddleNLP/unarchived/language_model/gru/_ce.py deleted file mode 100644 index d4999d7a1e14e333f1c7056b3dc2c5b506682ec6..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/language_model/gru/_ce.py +++ /dev/null @@ -1,62 +0,0 @@ -# this file is only used for continuous evaluation test! - -import os -import sys -sys.path.append(os.environ['ceroot']) -from kpi import CostKpi -from kpi import DurationKpi - -imikolov_20_avg_ppl_kpi = CostKpi('imikolov_20_avg_ppl', 0.2, 0) -imikolov_20_pass_duration_kpi = DurationKpi( - 'imikolov_20_pass_duration', 0.02, 0, actived=True) -imikolov_20_avg_ppl_kpi_card4 = CostKpi('imikolov_20_avg_ppl_card4', 0.2, 0) -imikolov_20_pass_duration_kpi_card4 = DurationKpi( - 'imikolov_20_pass_duration_card4', 0.03, 0, actived=True) - -tracking_kpis = [ - imikolov_20_avg_ppl_kpi, - imikolov_20_pass_duration_kpi, - imikolov_20_avg_ppl_kpi_card4, - imikolov_20_pass_duration_kpi_card4, -] - - -def parse_log(log): - ''' - This method should be implemented by model developers. - - The suggestion: - - each line in the log should be key, value, for example: - - " - train_cost\t1.0 - test_cost\t1.0 - train_cost\t1.0 - train_cost\t1.0 - train_acc\t1.2 - " - ''' - for line in log.split('\n'): - fs = line.strip().split('\t') - print(fs) - if len(fs) == 3 and fs[0] == 'kpis': - kpi_name = fs[1] - kpi_value = float(fs[2]) - yield kpi_name, kpi_value - - -def log_to_ce(log): - kpi_tracker = {} - for kpi in tracking_kpis: - kpi_tracker[kpi.name] = kpi - - for (kpi_name, kpi_value) in parse_log(log): - print(kpi_name, kpi_value) - kpi_tracker[kpi_name].add_record(kpi_value) - kpi_tracker[kpi_name].persist() - - -if __name__ == '__main__': - log = sys.stdin.read() - log_to_ce(log) diff --git a/PaddleNLP/unarchived/language_model/gru/infer.py b/PaddleNLP/unarchived/language_model/gru/infer.py deleted file mode 100644 index f10383afb22b3e01c4a353e651f8359dc636cf32..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/language_model/gru/infer.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import time -import math -import unittest -import contextlib -import numpy as np -import six - -import paddle -import paddle.fluid as fluid - -import utils - - -def infer(test_reader, use_cuda, model_path): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model( - model_path, exe) - - accum_cost = 0.0 - accum_words = 0 - t0 = time.time() - for data in test_reader(): - src_wordseq = utils.to_lodtensor([dat[0] for dat in data], place) - dst_wordseq = utils.to_lodtensor([dat[1] for dat in data], place) - avg_cost = exe.run( - infer_program, - feed={"src_wordseq": src_wordseq, - "dst_wordseq": dst_wordseq}, - fetch_list=fetch_vars) - - nwords = src_wordseq.lod()[0][-1] - - cost = np.array(avg_cost) * nwords - accum_cost += cost - accum_words += nwords - - ppl = math.exp(accum_cost / accum_words) - t1 = time.time() - print("model:%s ppl:%.3f time_cost(s):%.2f" % - (model_path, ppl, t1 - t0)) - - -if __name__ == "__main__": - if len(sys.argv) != 4: - print("Usage: %s model_dir start_epoch last_epoch(inclusive)") - exit(0) - - model_dir = sys.argv[1] - try: - start_index = int(sys.argv[2]) - last_index = int(sys.argv[3]) - except: - print("Usage: %s model_dir start_epoch last_epoch(inclusive)") - exit(-1) - - vocab, train_reader, test_reader = utils.prepare_data( - batch_size=20, buffer_size=1000, word_freq_threshold=0) - - for epoch in six.moves.xrange(start_index, last_index + 1): - epoch_path = model_dir + "/epoch_" + str(epoch) - infer(test_reader=test_reader, use_cuda=True, model_path=epoch_path) diff --git a/PaddleNLP/unarchived/language_model/gru/train.py b/PaddleNLP/unarchived/language_model/gru/train.py deleted file mode 100644 index 3e6183341b810c4bb9d7ad27d6b4562328552425..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/language_model/gru/train.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import sys -import time -import six - -import numpy as np -import math -import argparse -import paddle.fluid as fluid -import paddle - -import utils - -SEED = 102 - - -def parse_args(): - parser = argparse.ArgumentParser("language_model benchmark.") - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run \ - the task with continuous evaluation logs.') - parser.add_argument( - '--num_devices', type=int, default=1, help='Number of GPU devices') - args = parser.parse_args() - return args - - -def network(src, dst, vocab_size, hid_size, init_low_bound, init_high_bound): - """ network definition """ - emb_lr_x = 10.0 - gru_lr_x = 1.0 - fc_lr_x = 1.0 - emb = fluid.layers.embedding( - input=src, - size=[vocab_size, hid_size], - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=init_low_bound, high=init_high_bound), - learning_rate=emb_lr_x), - is_sparse=True) - - fc0 = fluid.layers.fc(input=emb, - size=hid_size * 3, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=init_low_bound, high=init_high_bound), - learning_rate=gru_lr_x)) - gru_h0 = fluid.layers.dynamic_gru( - input=fc0, - size=hid_size, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=init_low_bound, high=init_high_bound), - learning_rate=gru_lr_x)) - - fc = fluid.layers.fc(input=gru_h0, - size=vocab_size, - act='softmax', - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=init_low_bound, high=init_high_bound), - learning_rate=fc_lr_x)) - - cost = fluid.layers.cross_entropy(input=fc, label=dst) - return cost - - -def train(train_reader, - vocab, - network, - hid_size, - base_lr, - batch_size, - pass_num, - use_cuda, - parallel, - model_dir, - init_low_bound=-0.04, - init_high_bound=0.04): - """ train network """ - - args = parse_args() - if args.enable_ce: - # random seed must set before configuring the network. - fluid.default_startup_program().random_seed = SEED - vocab_size = len(vocab) - - #Input data - src_wordseq = fluid.layers.data( - name="src_wordseq", shape=[1], dtype="int64", lod_level=1) - dst_wordseq = fluid.layers.data( - name="dst_wordseq", shape=[1], dtype="int64", lod_level=1) - - # Train program - avg_cost = None - cost = network(src_wordseq, dst_wordseq, vocab_size, hid_size, - init_low_bound, init_high_bound) - avg_cost = fluid.layers.mean(x=cost) - - # Optimization to minimize lost - sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay( - learning_rate=base_lr, - decay_steps=2100 * 4, - decay_rate=0.5, - staircase=True)) - sgd_optimizer.minimize(avg_cost) - - # Initialize executor - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - exec_strategy = fluid.ExecutionStrategy() - exec_strategy.num_threads = 1 if os.name == 'nt' else 0 - train_exe = fluid.ParallelExecutor( - use_cuda=True, loss_name=avg_cost.name, exec_strategy=exec_strategy) - - total_time = 0.0 - fetch_list = [avg_cost.name] - for pass_idx in six.moves.xrange(pass_num): - epoch_idx = pass_idx + 1 - print("epoch_%d start" % epoch_idx) - - t0 = time.time() - i = 0 - newest_ppl = 0 - for data in train_reader(): - i += 1 - lod_src_wordseq = utils.to_lodtensor([dat[0] for dat in data], - place) - lod_dst_wordseq = utils.to_lodtensor([dat[1] for dat in data], - place) - ret_avg_cost = train_exe.run(feed={ - "src_wordseq": lod_src_wordseq, - "dst_wordseq": lod_dst_wordseq - }, - fetch_list=fetch_list) - avg_ppl = np.exp(ret_avg_cost[0]) - newest_ppl = np.mean(avg_ppl) - if i % 100 == 0: - print("step:%d ppl:%.3f" % (i, newest_ppl)) - - t1 = time.time() - total_time += t1 - t0 - print("epoch:%d num_steps:%d time_cost(s):%f" % - (epoch_idx, i, total_time / epoch_idx)) - - if pass_idx == pass_num - 1 and args.enable_ce: - #Note: The following logs are special for CE monitoring. - #Other situations do not need to care about these logs. - gpu_num = get_cards(args) - if gpu_num == 1: - print("kpis imikolov_20_pass_duration %s" % - (total_time / epoch_idx)) - print("kpis imikolov_20_avg_ppl %s" % newest_ppl) - else: - print("kpis imikolov_20_pass_duration_card%s %s" % \ - (gpu_num, total_time / epoch_idx)) - print("kpis imikolov_20_avg_ppl_card%s %s" % - (gpu_num, newest_ppl)) - save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) - feed_var_names = ["src_wordseq", "dst_wordseq"] - fetch_vars = [avg_cost] - fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe) - print("model saved in %s" % save_dir) - - print("finish training") - - -def get_cards(args): - if args.enable_ce: - cards = os.environ.get('CUDA_VISIBLE_DEVICES') - num = len(cards.split(",")) - return num - else: - return args.num_devices - - -def train_net(): - """ do training """ - batch_size = 20 - args = parse_args() - vocab, train_reader, test_reader = utils.prepare_data( - batch_size=batch_size * get_cards(args), buffer_size=1000, \ - word_freq_threshold=0, enable_ce = args.enable_ce) - train( - train_reader=train_reader, - vocab=vocab, - network=network, - hid_size=200, - base_lr=1.0, - batch_size=batch_size, - pass_num=12, - use_cuda=True, - parallel=True, - model_dir="model", - init_low_bound=-0.1, - init_high_bound=0.1) - - -if __name__ == "__main__": - train_net() diff --git a/PaddleNLP/unarchived/language_model/gru/train_on_cloud.py b/PaddleNLP/unarchived/language_model/gru/train_on_cloud.py deleted file mode 100644 index e5541ee462e4817facc77a4a4957ace85b3d53d2..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/language_model/gru/train_on_cloud.py +++ /dev/null @@ -1,301 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import sys -import time -import six - -import numpy as np -import math - -import collections -import paddle -import paddle.fluid as fluid -import paddle.fluid.framework as framework - -cluster_train_dir = "./train/" -cluster_test_dir = "./test/" -train_file = "ptb.train.txt" -valid_file = "ptb.valid.txt" -test_file = "ptb.test.txt" - - -class DataType(object): - """ data type """ - NGRAM = 1 - SEQ = 2 - - -def word_count(f, word_freq=None): - """ count words """ - if word_freq is None: - word_freq = collections.defaultdict(int) - - for line in f: - for w in line.strip().split(): - word_freq[w] += 1 - word_freq[''] += 1 - word_freq[''] += 1 - - return word_freq - - -def build_dict(min_word_freq=50): - """ build dictionary """ - train_filename = cluster_train_dir + train_file - test_filename = cluster_test_dir + valid_file - trainf = open(train_filename).readlines() - testf = open(test_filename).readlines() - word_freq = word_count(testf, word_count(trainf)) - if '' in word_freq: - del word_freq[''] - word_freq = filter(lambda x: x[1] > min_word_freq, word_freq.items()) - word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0])) - words, _ = list(zip(*word_freq_sorted)) - word_idx = dict(zip(words, six.moves.xrange(len(words)))) - word_idx[''] = len(words) - return word_idx - - -def reader_creator(filename, word_idx, n, data_type): - """ create reader """ - - def reader(): - if True: - f = open(filename).readlines() - UNK = word_idx[''] - for line in f: - if DataType.NGRAM == data_type: - assert n > -1, 'Invalid gram length' - line = [''] + line.strip().split() + [''] - if len(line) >= n: - line = [word_idx.get(w, UNK) for w in line] - for i in range(n, len(line) + 1): - yield tuple(line[i - n:i]) - elif DataType.SEQ == data_type: - line = line.strip().split() - line = [word_idx.get(w, UNK) for w in line] - src_seq = [word_idx['']] + line - trg_seq = line + [word_idx['']] - if n > 0 and len(src_seq) > n: - continue - yield src_seq, trg_seq - else: - assert False, 'Unknow data type' - - return reader - - -def to_lodtensor(data, place): - """ convert to LODtensor """ - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for line in seq_lens: - cur_len += line - lod.append(cur_len) - flattened_data = np.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - res = fluid.LoDTensor() - res.set(flattened_data, place) - res.set_lod([lod]) - return res - - -def prepare_data(batch_size, buffer_size=1000, word_freq_threshold=0): - """ prepare the English Pann Treebank (PTB) data """ - vocab = build_dict(word_freq_threshold) - train_reader = paddle.batch( - paddle.reader.shuffle( - reader_creator( - cluster_train_dir + train_file, - vocab, - buffer_size, - data_type=DataType.SEQ), - buf_size=buffer_size), - batch_size) - test_reader = paddle.batch( - reader_creator( - cluster_test_dir + test_file, - vocab, - buffer_size, - data_type=DataType.SEQ), - batch_size) - return vocab, train_reader, test_reader - - -def network(src, dst, vocab_size, hid_size, init_low_bound, init_high_bound): - """ network definition """ - emb_lr_x = 10.0 - gru_lr_x = 1.0 - fc_lr_x = 1.0 - emb = fluid.layers.embedding( - input=src, - size=[vocab_size, hid_size], - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=init_low_bound, high=init_high_bound), - learning_rate=emb_lr_x), - is_sparse=True) - - fc0 = fluid.layers.fc(input=emb, - size=hid_size * 3, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=init_low_bound, high=init_high_bound), - learning_rate=gru_lr_x)) - gru_h0 = fluid.layers.dynamic_gru( - input=fc0, - size=hid_size, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=init_low_bound, high=init_high_bound), - learning_rate=gru_lr_x)) - - fc = fluid.layers.fc(input=gru_h0, - size=vocab_size, - act='softmax', - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=init_low_bound, high=init_high_bound), - learning_rate=fc_lr_x)) - - cost = fluid.layers.cross_entropy(input=fc, label=dst) - return cost - - -def do_train(train_reader, - vocab, - network, - hid_size, - base_lr, - batch_size, - pass_num, - use_cuda, - parallel, - model_dir, - init_low_bound=-0.04, - init_high_bound=0.04): - """ train network """ - vocab_size = len(vocab) - - src_wordseq = fluid.layers.data( - name="src_wordseq", shape=[1], dtype="int64", lod_level=1) - dst_wordseq = fluid.layers.data( - name="dst_wordseq", shape=[1], dtype="int64", lod_level=1) - - avg_cost = None - if not parallel: - cost = network(src_wordseq, dst_wordseq, vocab_size, hid_size, - init_low_bound, init_high_bound) - avg_cost = fluid.layers.mean(x=cost) - else: - places = fluid.layers.device.get_places() - pd = fluid.layers.ParallelDo(places) - with pd.do(): - cost = network( - pd.read_input(src_wordseq), - pd.read_input(dst_wordseq), vocab_size, hid_size, - init_low_bound, init_high_bound) - pd.write_output(cost) - - cost = pd() - avg_cost = fluid.layers.mean(x=cost) - - sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay( - learning_rate=base_lr, - decay_steps=2100 * 4, - decay_rate=0.5, - staircase=True)) - sgd_optimizer.minimize(avg_cost) - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - - exe.run(fluid.default_startup_program()) - total_time = 0.0 - for pass_idx in six.moves.xrange(pass_num): - epoch_idx = pass_idx + 1 - print("epoch_%d start" % epoch_idx) - - t0 = time.time() - i = 0 - for data in train_reader(): - i += 1 - lod_src_wordseq = to_lodtensor([dat[0] for dat in data], place) - lod_dst_wordseq = to_lodtensor([dat[1] for dat in data], place) - ret_avg_cost = exe.run(fluid.default_main_program(), - feed={ - "src_wordseq": lod_src_wordseq, - "dst_wordseq": lod_dst_wordseq - }, - fetch_list=[avg_cost], - use_program_cache=True) - avg_ppl = math.exp(ret_avg_cost[0]) - if i % 100 == 0: - print("step:%d ppl:%.3f" % (i, avg_ppl)) - - t1 = time.time() - total_time += t1 - t0 - print("epoch:%d num_steps:%d time_cost(s):%f" % - (epoch_idx, i, total_time / epoch_idx)) - - save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) - feed_var_names = ["src_wordseq", "dst_wordseq"] - fetch_vars = [avg_cost] - fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe) - print("model saved in %s" % save_dir) - - print("finish training") - - -def train(): - """ do training """ - batch_size = 20 - vocab, train_reader, test_reader = prepare_data( - batch_size=batch_size, buffer_size=1000, word_freq_threshold=0) - - # End batch and end pass event handler - def event_handler(event): - """ event handler """ - if isinstance(event, paddle.event.EndIteration): - if event.batch_id % 100 == 0: - print("\nPass %d, Batch %d, Cost %f, %s" % ( - event.pass_id, event.batch_id, event.cost, event.metrics)) - else: - sys.stdout.write('.') - sys.stdout.flush() - if isinstance(event, paddle.event.EndPass): - print("isinstance(event, paddle.event.EndPass)") - - do_train( - train_reader=train_reader, - vocab=vocab, - network=network, - hid_size=200, - base_lr=1.0, - batch_size=batch_size, - pass_num=12, - use_cuda=True, - parallel=False, - model_dir="./output/model", - init_low_bound=-0.1, - init_high_bound=0.1) - - -if __name__ == "__main__": - if not os.path.exists("./output/model"): - os.makedirs("./output/model") - train() diff --git a/PaddleNLP/unarchived/language_model/gru/utils.py b/PaddleNLP/unarchived/language_model/gru/utils.py deleted file mode 100644 index 4535999d96addd9a414562bf5ffde44269d6cb0d..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/language_model/gru/utils.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import time -import numpy as np - -import paddle.fluid as fluid -import paddle - - -def to_lodtensor(data, place): - """ convert to LODtensor """ - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for l in seq_lens: - cur_len += l - lod.append(cur_len) - flattened_data = np.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - res = fluid.LoDTensor() - res.set(flattened_data, place) - res.set_lod([lod]) - return res - - -def prepare_data(batch_size, - buffer_size=1000, - word_freq_threshold=0, - enable_ce=False): - """ prepare the English Pann Treebank (PTB) data """ - vocab = paddle.dataset.imikolov.build_dict(word_freq_threshold) - if enable_ce: - train_reader = paddle.batch( - paddle.dataset.imikolov.train( - vocab, - buffer_size, - data_type=paddle.dataset.imikolov.DataType.SEQ), - batch_size) - else: - train_reader = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.imikolov.train( - vocab, - buffer_size, - data_type=paddle.dataset.imikolov.DataType.SEQ), - buf_size=buffer_size), - batch_size) - test_reader = paddle.batch( - paddle.dataset.imikolov.test( - vocab, buffer_size, data_type=paddle.dataset.imikolov.DataType.SEQ), - batch_size) - return vocab, train_reader, test_reader diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/.run_ce.sh b/PaddleNLP/unarchived/machine_reading_comprehension/.run_ce.sh deleted file mode 100644 index 22e0a73478b15e08b5e45b8086af9cadc8e74bc3..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/.run_ce.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -DATA_PATH=./data -if [ ! -e $DATA_PATH/demo ] ; then - mkdir -p $DATA_PATH/demo - if [ ! -e $DATA_PATH/demo.tgz ] ; then - cd $DATA_PATH - wget -c --no-check-certificate http://dureader.gz.bcebos.com/demo.tgz - cd - - fi - tar -zxf $DATA_PATH/demo.tgz -C $DATA_PATH/demo -fi - -train(){ -python -u run.py \ - --trainset 'data/demo/search.train.json' \ - --devset 'data/demo/search.dev.json' \ - --testset 'data/demo/search.test.json' \ - --vocab_dir 'data/demo/' \ - --use_gpu true \ - --save_dir ./models \ - --pass_num 1 \ - --learning_rate 0.001 \ - --batch_size 32 \ - --embed_size 300 \ - --hidden_size 150 \ - --max_p_num 5 \ - --max_p_len 500 \ - --max_q_len 60 \ - --max_a_len 200 \ - --drop_rate 0.2 \ - --log_interval 1 \ - --enable_ce \ - --train -} - -cudaid=${single:=0} # use 0-th card as default -export CUDA_VISIBLE_DEVICES=$cudaid - -train | python _ce.py - -cudaid=${multi:=0,1,2,3} # use 0,1,2,3 card as default -export CUDA_VISIBLE_DEVICES=$cudaid - -train | python _ce.py diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/README.md b/PaddleNLP/unarchived/machine_reading_comprehension/README.md deleted file mode 100644 index 3031511fb6af25e300a06440de633cac615269f4..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/README.md +++ /dev/null @@ -1,91 +0,0 @@ -DuReader是一个端到端的机器阅读理解神经网络模型，能够在给定文档和问题的情况下，定位文档中问题的答案。我们首先利用双向注意力网络获得文档和问题的相同向量空间的表示，然后使用`point network` 定位文档中答案的位置。实验显示，我们的模型能够获得在Dureader数据集上SOTA的结果。 - -# 算法介绍 -DuReader模型主要实现了论文[BiDAF](https://arxiv.org/abs/1611.01603)， [Match-LSTM](https://arxiv.org/abs/1608.07905)中的模型结构。 - -模型在层次上可以分为5层： - -- **词嵌入层** 将每一个词映射到一个向量(这个向量可以是预训练好的)。 -- **编码层** 使用双向LSMT网络获得问题和文档的每一个词的上下文信息。 -- **注意力层** 通过双向注意力网络获得文档的问题向量空间表示。更多参考[BiDAF](https://arxiv.org/abs/1611.01603)。 -- **融合层** 通过双向LSTM网络获得文档向量表示中上下文的关联性信息。 -- **输出层** 通过`point network`预测答案在问题中的位置。更多参考 [Match-LSTM](https://arxiv.org/abs/1608.07905)。 - -## 数据准备 -### 下载数据集 -通过如下脚本下载数据集: -``` -cd data && bash download.sh -``` -模型默认使用DuReader数据集，是百度开源的真实阅读理解数据，更多参考[DuReader Dataset Homepage](https://ai.baidu.com//broad/subordinate?dataset=dureader) - -### 下载第三方依赖 -我们使用Bleu和Rouge作为度量指标， 这些度量指标的源码位于[coco-caption](https://github.com/tylin/coco-caption)， 可以使用如下命令下载源码: - -``` -cd utils && bash download_thirdparty.sh -``` -### 环境依赖 -当前模型是在paddlepaddle 1.2版本上测试， 因此建议在1.2版本上使用本模型。关于PaddlePaddle的安装可以参考[PaddlePaddle Homepage](http://paddlepaddle.org)。 - -## 模型训练 -### 段落抽取 -在段落抽取阶段，主要是使用文档相关性score对文档内容进行优化， 抽取的结果将会放到`data/extracted/`目录下。如果你用demo数据测试，可以跳过这一步。如果你用dureader数据，需要指定抽取的数据目录，命令如下： -``` -bash run.sh --para_extraction --trainset data/preprocessed/trainset/zhidao.train.json data/preprocessed/trainset/search.train.json --devset data/preprocessed/devset/zhidao.dev.json data/preprocessed/devset/search.dev.json --testset data/preprocessed/testset/zhidao.test.json data/preprocessed/testset/search.test.json -``` -其中参数 `trainset`/`devset`/`testset`分别对应训练、验证和测试数据集(下同)。 -### 词典准备 -在训练模型之前，我们应该确保数据已经准备好。在准备阶段，通过全部数据文件生成一个词典，这个词典会在后续的训练和预测中用到。你可以通过如下命令生成词典： -``` -bash run.sh --prepare -``` -上面的命令默认使用demo数据，如果想使用dureader数据集，应该按照如下方式指定： -``` -bash run.sh --prepare --trainset data/extracted/trainset/zhidao.train.json data/extracted/trainset/search.train.json --devset data/extracted/devset/zhidao.dev.json data/extracted/devset/search.dev.json --testset data/extracted/testset/zhidao.test.json data/extracted/testset/search.test.json -``` -其中参数 `trainset`/`devset`/`testset`分别对应训练、验证和测试数据集。 -### 模型训练 -训练模型的启动命令如下： -``` -bash run.sh --train -``` -上面的命令默认使用demo数据，如果想使用dureader数据集，应该按照如下方式指定： -``` -bash run.sh --train --trainset data/extracted/trainset/zhidao.train.json data/extracted/trainset/search.train.json --devset data/extracted/devset/zhidao.dev.json data/extracted/devset/search.dev.json --testset data/extracted/testset/zhidao.test.json data/extracted/testset/search.test.json -``` -可以通过设置超参数更改训练的配置，比如通过`--learning_rate NUM`更改学习率，通过`--pass_num NUM`更改训练的轮数 -训练的过程中，每隔一定迭代周期，会测试在验证集上的性能指标， 通过`--dev_interval NUM`设置周期大小 - -### 模型评测 -在模型训练结束后，如果想使用训练好的模型进行评测，获得度量指标，可以使用如下命令: -``` -bash run.sh --evaluate --load_dir data/models/1 -``` -其中，`--load_dir data/models/1`是模型的checkpoint目录 - -上面的命令默认使用demo数据，如果想使用dureader数据集，应该按照如下方式指定： -``` -bash run.sh --evaluate --load_dir data/models/1 --devset data/extracted/devset/zhidao.dev.json data/extracted/devset/search.dev.json --testset data/extracted/testset/zhidao.test.json data/extracted/testset/search.test.json -``` - -### 预测 -使用训练好的模型，对问答文档数据直接预测结果，获得答案，可以使用如下命令: -``` -bash run.sh --predict --load_dir data/models/1 -``` -上面的命令默认使用demo数据，如果想使用dureader数据集，应该按照如下方式指定： -``` -bash run.sh --predict --load_dir data/models/1 --testset data/extracted/testset/search.test.json data/extracted/testset/zhidao.test.json -``` -其中`--testset`指定了预测用的数据集，生成的问题答案默认会放到`data/results/` 目录，你可以通过参数`--result_dir DIR_PATH`更改配置 - -### 实验结果 -验证集 ROUGE-L:47.65。 - -这是在P40上，使用4卡GPU，batch size=4*32的训练5个epoch(约30个小时)的结果，如果使用单卡，指标可能会略有降低，但在验证集上的ROUGE-L也不小于47。 - -## 参考文献 -[Machine Comprehension Using Match-LSTM and Answer Pointer](https://arxiv.org/abs/1608.07905) - -[Bidirectional Attention Flow for Machine Comprehension](https://arxiv.org/abs/1611.01603) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/_ce.py b/PaddleNLP/unarchived/machine_reading_comprehension/_ce.py deleted file mode 100644 index a425fe951fb587749f31b18959917cdeed76a41d..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/_ce.py +++ /dev/null @@ -1,69 +0,0 @@ -####this file is only used for continuous evaluation test! - -import os -import sys -#sys.path.insert(0, os.environ['ceroot']) -sys.path.append(os.environ['ceroot']) -from kpi import CostKpi, DurationKpi, AccKpi - -#### NOTE kpi.py should shared in models in some way!!!! - -train_cost_card1_kpi = CostKpi('train_cost_card1', 0.02, 0, actived=True) -test_cost_card1_kpi = CostKpi('test_cost_card1', 0.005, 0, actived=True) -train_duration_card1_kpi = DurationKpi( - 'train_duration_card1', 0.06, 0, actived=True) -train_cost_card4_kpi = CostKpi('train_cost_card4', 0.01, 0, actived=True) -test_cost_card4_kpi = CostKpi('test_cost_card4', 0.005, 0, actived=True) -train_duration_card4_kpi = DurationKpi( - 'train_duration_card4', 0.06, 0, actived=True) - -tracking_kpis = [ - train_cost_card1_kpi, - test_cost_card1_kpi, - train_duration_card1_kpi, - train_cost_card4_kpi, - test_cost_card4_kpi, - train_duration_card4_kpi, -] - - -def parse_log(log): - ''' - This method should be implemented by model developers. - The suggestion: - each line in the log should be key, value, for example: - " - train_cost\t1.0 - test_cost\t1.0 - train_cost\t1.0 - train_cost\t1.0 - train_acc\t1.2 - " - ''' - for line in log.split('\n'): - fs = line.strip().split('\t') - print(fs) - if len(fs) == 3 and fs[0] == 'kpis': - print("-----%s" % fs) - kpi_name = fs[1] - kpi_value = float(fs[2]) - yield kpi_name, kpi_value - - -def log_to_ce(log): - kpi_tracker = {} - for kpi in tracking_kpis: - kpi_tracker[kpi.name] = kpi - - for (kpi_name, kpi_value) in parse_log(log): - print(kpi_name, kpi_value) - kpi_tracker[kpi_name].add_record(kpi_value) - kpi_tracker[kpi_name].persist() - - -if __name__ == '__main__': - log = sys.stdin.read() - print("*****") - print(log) - print("****") - log_to_ce(log) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/args.py b/PaddleNLP/unarchived/machine_reading_comprehension/args.py deleted file mode 100644 index 81829c712e76d59ed4e25ddfe6b379782275db8b..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/args.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import distutils.util - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - '--prepare', - action='store_true', - help='create the directories, prepare the vocabulary and embeddings') - parser.add_argument('--train', action='store_true', help='train the model') - parser.add_argument( - '--evaluate', action='store_true', help='evaluate the model on dev set') - parser.add_argument( - '--predict', - action='store_true', - help='predict the answers for test set with trained model') - - parser.add_argument( - "--embed_size", - type=int, - default=300, - help="The dimension of embedding table. (default: %(default)d)") - parser.add_argument( - "--hidden_size", - type=int, - default=150, - help="The size of rnn hidden unit. (default: %(default)d)") - parser.add_argument( - "--learning_rate", - type=float, - default=0.001, - help="Learning rate used to train the model. (default: %(default)f)") - parser.add_argument('--optim', default='adam', help='optimizer type') - parser.add_argument( - "--weight_decay", - type=float, - default=0.0001, - help="Weight decay. (default: %(default)f)") - - parser.add_argument( - '--drop_rate', type=float, default=0.0, help="Dropout probability") - parser.add_argument('--random_seed', type=int, default=123) - parser.add_argument( - "--batch_size", - type=int, - default=32, - help="The sequence number of a mini-batch data. (default: %(default)d)") - parser.add_argument( - "--pass_num", - type=int, - default=5, - help="The number epochs to train. (default: %(default)d)") - parser.add_argument( - "--use_gpu", - type=distutils.util.strtobool, - default=True, - help="Whether to use gpu. (default: %(default)d)") - parser.add_argument( - "--log_interval", - type=int, - default=50, - help="log the train loss every n batches. (default: %(default)d)") - - parser.add_argument('--max_p_num', type=int, default=5) - parser.add_argument('--max_a_len', type=int, default=200) - parser.add_argument('--max_p_len', type=int, default=500) - parser.add_argument('--max_q_len', type=int, default=60) - parser.add_argument('--doc_num', type=int, default=5) - - parser.add_argument('--vocab_dir', default='data/vocab', help='vocabulary') - parser.add_argument( - "--save_dir", - type=str, - default="data/models", - help="Specify the path to save trained models.") - parser.add_argument( - "--save_interval", - type=int, - default=1, - help="Save the trained model every n passes. (default: %(default)d)") - parser.add_argument( - "--load_dir", - type=str, - default="", - help="Specify the path to load trained models.") - parser.add_argument( - '--log_path', - help='path of the log file. If not set, logs are printed to console') - parser.add_argument( - '--result_dir', - default='data/results/', - help='the dir to output the results') - parser.add_argument( - '--result_name', - default='test_result', - help='the file name of the predicted results') - - parser.add_argument( - '--trainset', - nargs='+', - default=['data/demo/trainset/search.train.json'], - help='train dataset') - parser.add_argument( - '--devset', - nargs='+', - default=['data/demo/devset/search.dev.json'], - help='dev dataset') - parser.add_argument( - '--testset', - nargs='+', - default=['data/demo/testset/search.test.json'], - help='test dataset') - - parser.add_argument( - "--enable_ce", - action='store_true', - help="If set, run the task with continuous evaluation logs.") - parser.add_argument( - '--para_print', action='store_true', help="Print debug info") - parser.add_argument( - "--dev_interval", - type=int, - default=-1, - help="evaluate on dev set loss every n batches. (default: %(default)d)") - args = parser.parse_args() - return args diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/data/download.sh b/PaddleNLP/unarchived/machine_reading_comprehension/data/download.sh deleted file mode 100644 index 5ad22e5c3de8c3e36e6105f00de2dfa7eeb46af3..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/data/download.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# ============================================================================== -# Copyright 2017 Baidu.com, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - - -if [[ -d preprocessed ]] && [[ -d raw ]]; then - echo "data exist" - exit 0 -else - wget -c http://dureader.gz.bcebos.com/demo.zip - wget -c https://aipedataset.cdn.bcebos.com/dureader/dureader_raw.zip - wget -c https://aipedataset.cdn.bcebos.com/dureader/dureader_preprocessed.zip -fi - -if md5sum --status -c md5sum.txt; then - unzip demo.zip - unzip dureader_raw.zip - unzip dureader_preprocessed.zip -else - echo "download data error!" >> /dev/stderr - exit 1 -fi diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/data/md5sum.txt b/PaddleNLP/unarchived/machine_reading_comprehension/data/md5sum.txt deleted file mode 100644 index 118e34190c5823c0d096ff5c300bf877101679d4..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/data/md5sum.txt +++ /dev/null @@ -1,3 +0,0 @@ -0ca0510fa625d35d902b73033c4ba9d8 demo.zip -dc7658b8cdf4f94b8714d130b7d15196 dureader_raw.zip -3db9a32e5a7c5375a604a70687b45479 dureader_preprocessed.zip diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/dataset.py b/PaddleNLP/unarchived/machine_reading_comprehension/dataset.py deleted file mode 100644 index 7c583e100997b41aa5d5229c4c3f8fee59d7539f..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/dataset.py +++ /dev/null @@ -1,244 +0,0 @@ -# -*- coding:utf8 -*- -# ============================================================================== -# Copyright 2017 Baidu.com, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -""" -This module implements data process strategies. -""" - -import os -import json -import logging -import numpy as np -from collections import Counter -import io - - -class BRCDataset(object): - """ - This module implements the APIs for loading and using baidu reading comprehension dataset - """ - - def __init__(self, - max_p_num, - max_p_len, - max_q_len, - train_files=[], - dev_files=[], - test_files=[]): - self.logger = logging.getLogger("brc") - self.max_p_num = max_p_num - self.max_p_len = max_p_len - self.max_q_len = max_q_len - - self.train_set, self.dev_set, self.test_set = [], [], [] - if train_files: - for train_file in train_files: - self.train_set += self._load_dataset(train_file, train=True) - self.logger.info('Train set size: {} questions.'.format( - len(self.train_set))) - - if dev_files: - for dev_file in dev_files: - self.dev_set += self._load_dataset(dev_file) - self.logger.info('Dev set size: {} questions.'.format( - len(self.dev_set))) - - if test_files: - for test_file in test_files: - self.test_set += self._load_dataset(test_file) - self.logger.info('Test set size: {} questions.'.format( - len(self.test_set))) - - def _load_dataset(self, data_path, train=False): - """ - Loads the dataset - Args: - data_path: the data file to load - """ - with io.open(data_path, 'r', encoding='utf-8') as fin: - data_set = [] - for lidx, line in enumerate(fin): - sample = json.loads(line.strip()) - if train: - if len(sample['answer_spans']) == 0: - continue - if sample['answer_spans'][0][1] >= self.max_p_len: - continue - - if 'answer_docs' in sample: - sample['answer_passages'] = sample['answer_docs'] - - sample['question_tokens'] = sample['segmented_question'] - - sample['passages'] = [] - for d_idx, doc in enumerate(sample['documents']): - if train: - most_related_para = doc['most_related_para'] - sample['passages'].append({ - 'passage_tokens': - doc['segmented_paragraphs'][most_related_para], - 'is_selected': doc['is_selected'] - }) - else: - para_infos = [] - for para_tokens in doc['segmented_paragraphs']: - question_tokens = sample['segmented_question'] - common_with_question = Counter( - para_tokens) & Counter(question_tokens) - correct_preds = sum(common_with_question.values()) - if correct_preds == 0: - recall_wrt_question = 0 - else: - recall_wrt_question = float( - correct_preds) / len(question_tokens) - para_infos.append((para_tokens, recall_wrt_question, - len(para_tokens))) - para_infos.sort(key=lambda x: (-x[1], x[2])) - fake_passage_tokens = [] - for para_info in para_infos[:1]: - fake_passage_tokens += para_info[0] - sample['passages'].append({ - 'passage_tokens': fake_passage_tokens - }) - data_set.append(sample) - return data_set - - def _one_mini_batch(self, data, indices, pad_id): - """ - Get one mini batch - Args: - data: all data - indices: the indices of the samples to be selected - pad_id: - Returns: - one batch of data - """ - batch_data = { - 'raw_data': [data[i] for i in indices], - 'question_token_ids': [], - 'question_length': [], - 'passage_token_ids': [], - 'passage_length': [], - 'start_id': [], - 'end_id': [], - 'passage_num': [] - } - max_passage_num = max( - [len(sample['passages']) for sample in batch_data['raw_data']]) - max_passage_num = min(self.max_p_num, max_passage_num) - for sidx, sample in enumerate(batch_data['raw_data']): - count = 0 - for pidx in range(max_passage_num): - if pidx < len(sample['passages']): - count += 1 - batch_data['question_token_ids'].append(sample[ - 'question_token_ids'][0:self.max_q_len]) - batch_data['question_length'].append( - min(len(sample['question_token_ids']), self.max_q_len)) - passage_token_ids = sample['passages'][pidx][ - 'passage_token_ids'][0:self.max_p_len] - batch_data['passage_token_ids'].append(passage_token_ids) - batch_data['passage_length'].append( - min(len(passage_token_ids), self.max_p_len)) - # record the start passage index of current sample - passade_idx_offset = sum(batch_data['passage_num']) - batch_data['passage_num'].append(count) - gold_passage_offset = 0 - if 'answer_passages' in sample and len(sample['answer_passages']) and \ - sample['answer_passages'][0] < len(sample['documents']): - for i in range(sample['answer_passages'][0]): - gold_passage_offset += len(batch_data['passage_token_ids'][ - passade_idx_offset + i]) - start_id = min(sample['answer_spans'][0][0], self.max_p_len) - end_id = min(sample['answer_spans'][0][1], self.max_p_len) - batch_data['start_id'].append(gold_passage_offset + start_id) - batch_data['end_id'].append(gold_passage_offset + end_id) - else: - # fake span for some samples, only valid for testing - batch_data['start_id'].append(0) - batch_data['end_id'].append(0) - return batch_data - - def word_iter(self, set_name=None): - """ - Iterates over all the words in the dataset - Args: - set_name: if it is set, then the specific set will be used - Returns: - a generator - """ - if set_name is None: - data_set = self.train_set + self.dev_set + self.test_set - elif set_name == 'train': - data_set = self.train_set - elif set_name == 'dev': - data_set = self.dev_set - elif set_name == 'test': - data_set = self.test_set - else: - raise NotImplementedError('No data set named as {}'.format( - set_name)) - if data_set is not None: - for sample in data_set: - for token in sample['question_tokens']: - yield token - for passage in sample['passages']: - for token in passage['passage_tokens']: - yield token - - def convert_to_ids(self, vocab): - """ - Convert the question and passage in the original dataset to ids - Args: - vocab: the vocabulary on this dataset - """ - for data_set in [self.train_set, self.dev_set, self.test_set]: - if data_set is None: - continue - for sample in data_set: - sample['question_token_ids'] = vocab.convert_to_ids(sample[ - 'question_tokens']) - for passage in sample['passages']: - passage['passage_token_ids'] = vocab.convert_to_ids(passage[ - 'passage_tokens']) - - def gen_mini_batches(self, set_name, batch_size, pad_id, shuffle=True): - """ - Generate data batches for a specific dataset (train/dev/test) - Args: - set_name: train/dev/test to indicate the set - batch_size: number of samples in one batch - pad_id: pad id - shuffle: if set to be true, the data is shuffled. - Returns: - a generator for all batches - """ - if set_name == 'train': - data = self.train_set - elif set_name == 'dev': - data = self.dev_set - elif set_name == 'test': - data = self.test_set - else: - raise NotImplementedError('No data set named as {}'.format( - set_name)) - data_size = len(data) - indices = np.arange(data_size) - if shuffle: - np.random.shuffle(indices) - for batch_start in np.arange(0, data_size, batch_size): - batch_indices = indices[batch_start:batch_start + batch_size] - yield self._one_mini_batch(data, batch_indices, pad_id) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/paragraph_extraction.py b/PaddleNLP/unarchived/machine_reading_comprehension/paragraph_extraction.py deleted file mode 100644 index 6729b84df294c1f86fd40c23dfa75e531d322872..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/paragraph_extraction.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#!/usr/bin/python -#-*- coding:utf-8 -*- - -import sys -if sys.version[0] == '2': - reload(sys) - sys.setdefaultencoding("utf-8") -import json -import copy -from preprocess import metric_max_over_ground_truths, f1_score - - -def compute_paragraph_score(sample): - """ - For each paragraph, compute the f1 score compared with the question - Args: - sample: a sample in the dataset. - Returns: - None - Raises: - None - """ - question = sample["segmented_question"] - for doc in sample['documents']: - doc['segmented_paragraphs_scores'] = [] - for p_idx, para_tokens in enumerate(doc['segmented_paragraphs']): - if len(question) > 0: - related_score = metric_max_over_ground_truths( - f1_score, para_tokens, question) - else: - related_score = 0.0 - doc['segmented_paragraphs_scores'].append(related_score) - - -def dup_remove(doc): - """ - For each document, remove the duplicated paragraphs - Args: - doc: a doc in the sample - Returns: - bool - Raises: - None - """ - paragraphs_his = {} - del_ids = [] - para_id = None - if 'most_related_para' in doc: - para_id = doc['most_related_para'] - doc['paragraphs_length'] = [] - for p_idx, (segmented_paragraph, paragraph_score) in \ - enumerate(zip(doc["segmented_paragraphs"], doc["segmented_paragraphs_scores"])): - doc['paragraphs_length'].append(len(segmented_paragraph)) - paragraph = ''.join(segmented_paragraph) - if paragraph in paragraphs_his: - del_ids.append(p_idx) - if p_idx == para_id: - para_id = paragraphs_his[paragraph] - continue - paragraphs_his[paragraph] = p_idx - # delete - prev_del_num = 0 - del_num = 0 - for p_idx in del_ids: - if p_idx < para_id: - prev_del_num += 1 - del doc["segmented_paragraphs"][p_idx - del_num] - del doc["segmented_paragraphs_scores"][p_idx - del_num] - del doc['paragraphs_length'][p_idx - del_num] - del_num += 1 - if len(del_ids) != 0: - if 'most_related_para' in doc: - doc['most_related_para'] = para_id - prev_del_num - doc['paragraphs'] = [] - for segmented_para in doc["segmented_paragraphs"]: - paragraph = ''.join(segmented_para) - doc['paragraphs'].append(paragraph) - return True - else: - return False - - -def paragraph_selection(sample, mode): - """ - For each document, select paragraphs that includes as much information as possible - Args: - sample: a sample in the dataset. - mode: string of ("train", "dev", "test"), indicate the type of dataset to process. - Returns: - None - Raises: - None - """ - # predefined maximum length of paragraph - MAX_P_LEN = 500 - # predefined splitter - splitter = u'' - # topN of related paragraph to choose - topN = 3 - doc_id = None - if 'answer_docs' in sample and len(sample['answer_docs']) > 0: - doc_id = sample['answer_docs'][0] - if doc_id >= len(sample['documents']): - # Data error, answer doc ID > number of documents, this sample - # will be filtered by dataset.py - return - for d_idx, doc in enumerate(sample['documents']): - if 'segmented_paragraphs_scores' not in doc: - continue - status = dup_remove(doc) - segmented_title = doc["segmented_title"] - title_len = len(segmented_title) - para_id = None - if doc_id is not None: - para_id = sample['documents'][doc_id]['most_related_para'] - total_len = title_len + sum(doc['paragraphs_length']) - # add splitter - para_num = len(doc["segmented_paragraphs"]) - total_len += para_num - if total_len <= MAX_P_LEN: - incre_len = title_len - total_segmented_content = copy.deepcopy(segmented_title) - for p_idx, segmented_para in enumerate(doc["segmented_paragraphs"]): - if doc_id == d_idx and para_id > p_idx: - incre_len += len([splitter] + segmented_para) - if doc_id == d_idx and para_id == p_idx: - incre_len += 1 - total_segmented_content += [splitter] + segmented_para - if doc_id == d_idx: - answer_start = incre_len + sample['answer_spans'][0][0] - answer_end = incre_len + sample['answer_spans'][0][1] - sample['answer_spans'][0][0] = answer_start - sample['answer_spans'][0][1] = answer_end - doc["segmented_paragraphs"] = [total_segmented_content] - doc["segmented_paragraphs_scores"] = [1.0] - doc['paragraphs_length'] = [total_len] - doc['paragraphs'] = [''.join(total_segmented_content)] - doc['most_related_para'] = 0 - continue - # find topN paragraph id - para_infos = [] - for p_idx, (para_tokens, para_scores) in \ - enumerate(zip(doc['segmented_paragraphs'], doc['segmented_paragraphs_scores'])): - para_infos.append( - (para_tokens, para_scores, len(para_tokens), p_idx)) - para_infos.sort(key=lambda x: (-x[1], x[2])) - topN_idx = [] - for para_info in para_infos[:topN]: - topN_idx.append(para_info[-1]) - final_idx = [] - total_len = title_len - if doc_id == d_idx: - if mode == "train": - final_idx.append(para_id) - total_len = title_len + 1 + doc['paragraphs_length'][para_id] - for id in topN_idx: - if total_len > MAX_P_LEN: - break - if doc_id == d_idx and id == para_id and mode == "train": - continue - total_len += 1 + doc['paragraphs_length'][id] - final_idx.append(id) - total_segmented_content = copy.deepcopy(segmented_title) - final_idx.sort() - incre_len = title_len - for id in final_idx: - if doc_id == d_idx and id < para_id: - incre_len += 1 + doc['paragraphs_length'][id] - if doc_id == d_idx and id == para_id: - incre_len += 1 - total_segmented_content += [splitter] + doc['segmented_paragraphs'][ - id] - if doc_id == d_idx: - answer_start = incre_len + sample['answer_spans'][0][0] - answer_end = incre_len + sample['answer_spans'][0][1] - sample['answer_spans'][0][0] = answer_start - sample['answer_spans'][0][1] = answer_end - doc["segmented_paragraphs"] = [total_segmented_content] - doc["segmented_paragraphs_scores"] = [1.0] - doc['paragraphs_length'] = [total_len] - doc['paragraphs'] = [''.join(total_segmented_content)] - doc['most_related_para'] = 0 - - -if __name__ == "__main__": - # mode="train"/"dev"/"test" - mode = sys.argv[1] - for line in sys.stdin: - line = line.strip() - if line == "": - continue - try: - sample = json.loads(line, encoding='utf8') - except: - print >> sys.stderr, "Invalid input json format - '{}' will be ignored".format( - line) - continue - compute_paragraph_score(sample) - paragraph_selection(sample, mode) - print(json.dumps(sample, encoding='utf8', ensure_ascii=False)) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/preprocess.py b/PaddleNLP/unarchived/machine_reading_comprehension/preprocess.py deleted file mode 100644 index 65842e7dd5fad312627292756dfcb4d7f026d3b3..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/preprocess.py +++ /dev/null @@ -1,219 +0,0 @@ -############################################################################### -# ============================================================================== -# Copyright 2017 Baidu.com, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -""" -This module finds the most related paragraph of each document according to recall. -""" - -import sys -if sys.version[0] == '2': - reload(sys) - sys.setdefaultencoding("utf-8") -import json -from collections import Counter - - -def precision_recall_f1(prediction, ground_truth): - """ - This function calculates and returns the precision, recall and f1-score - Args: - prediction: prediction string or list to be matched - ground_truth: golden string or list reference - Returns: - floats of (p, r, f1) - Raises: - None - """ - if not isinstance(prediction, list): - prediction_tokens = prediction.split() - else: - prediction_tokens = prediction - if not isinstance(ground_truth, list): - ground_truth_tokens = ground_truth.split() - else: - ground_truth_tokens = ground_truth - common = Counter(prediction_tokens) & Counter(ground_truth_tokens) - num_same = sum(common.values()) - if num_same == 0: - return 0, 0, 0 - p = 1.0 * num_same / len(prediction_tokens) - r = 1.0 * num_same / len(ground_truth_tokens) - f1 = (2 * p * r) / (p + r) - return p, r, f1 - - -def recall(prediction, ground_truth): - """ - This function calculates and returns the recall - Args: - prediction: prediction string or list to be matched - ground_truth: golden string or list reference - Returns: - floats of recall - Raises: - None - """ - return precision_recall_f1(prediction, ground_truth)[1] - - -def f1_score(prediction, ground_truth): - """ - This function calculates and returns the f1-score - Args: - prediction: prediction string or list to be matched - ground_truth: golden string or list reference - Returns: - floats of f1 - Raises: - None - """ - return precision_recall_f1(prediction, ground_truth)[2] - - -def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): - """ - This function calculates and returns the precision, recall and f1-score - Args: - metric_fn: metric function pointer which calculates scores according to corresponding logic. - prediction: prediction string or list to be matched - ground_truth: golden string or list reference - Returns: - floats of (p, r, f1) - Raises: - None - """ - scores_for_ground_truths = [] - for ground_truth in ground_truths: - score = metric_fn(prediction, ground_truth) - scores_for_ground_truths.append(score) - return max(scores_for_ground_truths) - - -def find_best_question_match(doc, question, with_score=False): - """ - For each document, find the paragraph that matches best to the question. - Args: - doc: The document object. - question: The question tokens. - with_score: If True then the match score will be returned, - otherwise False. - Returns: - The index of the best match paragraph, if with_score=False, - otherwise returns a tuple of the index of the best match paragraph - and the match score of that paragraph. - """ - most_related_para = -1 - max_related_score = 0 - most_related_para_len = 0 - for p_idx, para_tokens in enumerate(doc['segmented_paragraphs']): - if len(question) > 0: - related_score = metric_max_over_ground_truths(recall, para_tokens, - question) - else: - related_score = 0 - - if related_score > max_related_score \ - or (related_score == max_related_score \ - and len(para_tokens) < most_related_para_len): - most_related_para = p_idx - max_related_score = related_score - most_related_para_len = len(para_tokens) - if most_related_para == -1: - most_related_para = 0 - if with_score: - return most_related_para, max_related_score - return most_related_para - - -def find_fake_answer(sample): - """ - For each document, finds the most related paragraph based on recall, - then finds a span that maximize the f1_score compared with the gold answers - and uses this span as a fake answer span - Args: - sample: a sample in the dataset - Returns: - None - Raises: - None - """ - for doc in sample['documents']: - most_related_para = -1 - most_related_para_len = 999999 - max_related_score = 0 - for p_idx, para_tokens in enumerate(doc['segmented_paragraphs']): - if len(sample['segmented_answers']) > 0: - related_score = metric_max_over_ground_truths( - recall, para_tokens, sample['segmented_answers']) - else: - continue - if related_score > max_related_score \ - or (related_score == max_related_score - and len(para_tokens) < most_related_para_len): - most_related_para = p_idx - most_related_para_len = len(para_tokens) - max_related_score = related_score - doc['most_related_para'] = most_related_para - - sample['answer_docs'] = [] - sample['answer_spans'] = [] - sample['fake_answers'] = [] - sample['match_scores'] = [] - - best_match_score = 0 - best_match_d_idx, best_match_span = -1, [-1, -1] - best_fake_answer = None - answer_tokens = set() - for segmented_answer in sample['segmented_answers']: - answer_tokens = answer_tokens | set( - [token for token in segmented_answer]) - for d_idx, doc in enumerate(sample['documents']): - if not doc['is_selected']: - continue - if doc['most_related_para'] == -1: - doc['most_related_para'] = 0 - most_related_para_tokens = doc['segmented_paragraphs'][doc[ - 'most_related_para']][:1000] - for start_tidx in range(len(most_related_para_tokens)): - if most_related_para_tokens[start_tidx] not in answer_tokens: - continue - for end_tidx in range( - len(most_related_para_tokens) - 1, start_tidx - 1, -1): - span_tokens = most_related_para_tokens[start_tidx:end_tidx + 1] - if len(sample['segmented_answers']) > 0: - match_score = metric_max_over_ground_truths( - f1_score, span_tokens, sample['segmented_answers']) - else: - match_score = 0 - if match_score == 0: - break - if match_score > best_match_score: - best_match_d_idx = d_idx - best_match_span = [start_tidx, end_tidx] - best_match_score = match_score - best_fake_answer = ''.join(span_tokens) - if best_match_score > 0: - sample['answer_docs'].append(best_match_d_idx) - sample['answer_spans'].append(best_match_span) - sample['fake_answers'].append(best_fake_answer) - sample['match_scores'].append(best_match_score) - - -if __name__ == '__main__': - for line in sys.stdin: - sample = json.loads(line) - find_fake_answer(sample) - print(json.dumps(sample, encoding='utf8', ensure_ascii=False)) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/rc_model.py b/PaddleNLP/unarchived/machine_reading_comprehension/rc_model.py deleted file mode 100644 index edc009c26f8554c63964d09db59d45e65505b46a..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/rc_model.py +++ /dev/null @@ -1,331 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import paddle.fluid.layers as layers -import paddle.fluid as fluid -import numpy as np - - -def dropout(input, args): - """Dropout function""" - if args.drop_rate: - return layers.dropout( - input, - dropout_prob=args.drop_rate, - seed=args.random_seed, - is_test=False) - else: - return input - - -def bi_lstm_encoder(input_seq, gate_size, para_name, args): - """ - A bi-directional lstm encoder implementation. - Linear transformation part for input gate, output gate, forget gate - and cell activation vectors need be done outside of dynamic_lstm. - So the output size is 4 times of gate_size. - """ - - input_forward_proj = layers.fc( - input=input_seq, - param_attr=fluid.ParamAttr(name=para_name + '_fw_gate_w'), - size=gate_size * 4, - act=None, - bias_attr=False) - input_reversed_proj = layers.fc( - input=input_seq, - param_attr=fluid.ParamAttr(name=para_name + '_bw_gate_w'), - size=gate_size * 4, - act=None, - bias_attr=False) - forward, _ = layers.dynamic_lstm( - input=input_forward_proj, - size=gate_size * 4, - use_peepholes=False, - param_attr=fluid.ParamAttr(name=para_name + '_fw_lstm_w'), - bias_attr=fluid.ParamAttr(name=para_name + '_fw_lstm_b')) - reversed, _ = layers.dynamic_lstm( - input=input_reversed_proj, - param_attr=fluid.ParamAttr(name=para_name + '_bw_lstm_w'), - bias_attr=fluid.ParamAttr(name=para_name + '_bw_lstm_b'), - size=gate_size * 4, - is_reverse=True, - use_peepholes=False) - - encoder_out = layers.concat(input=[forward, reversed], axis=1) - return encoder_out - - -def get_data(input_name, lod_level, args): - input_ids = layers.data( - name=input_name, shape=[1], dtype='int64', lod_level=lod_level) - return input_ids - - -def embedding(input_ids, shape, args): - """Embedding layer""" - input_embedding = layers.embedding( - input=input_ids, - size=shape, - dtype='float32', - is_sparse=True, - param_attr=fluid.ParamAttr(name='embedding_para')) - return input_embedding - - -def encoder(input_embedding, para_name, hidden_size, args): - """Encoding layer""" - encoder_out = bi_lstm_encoder( - input_seq=input_embedding, - gate_size=hidden_size, - para_name=para_name, - args=args) - return dropout(encoder_out, args) - - -def attn_flow(q_enc, p_enc, p_ids_name, args): - """Bidirectional Attention layer""" - tag = p_ids_name + "::" - drnn = layers.DynamicRNN() - with drnn.block(): - h_cur = drnn.step_input(p_enc) - u_all = drnn.static_input(q_enc) - h_expd = layers.sequence_expand(x=h_cur, y=u_all) - s_t_mul = layers.elementwise_mul(x=u_all, y=h_expd, axis=0) - s_t_sum = layers.reduce_sum(input=s_t_mul, dim=1, keep_dim=True) - s_t_re = layers.reshape(s_t_sum, shape=[-1, 0]) - s_t = layers.sequence_softmax(input=s_t_re) - u_expr = layers.elementwise_mul(x=u_all, y=s_t, axis=0) - u_expr = layers.sequence_pool(input=u_expr, pool_type='sum') - - b_t = layers.sequence_pool(input=s_t_sum, pool_type='max') - drnn.output(u_expr, b_t) - U_expr, b = drnn() - b_norm = layers.sequence_softmax(input=b) - h_expr = layers.elementwise_mul(x=p_enc, y=b_norm, axis=0) - h_expr = layers.sequence_pool(input=h_expr, pool_type='sum') - - H_expr = layers.sequence_expand(x=h_expr, y=p_enc) - H_expr = layers.lod_reset(x=H_expr, y=p_enc) - h_u = layers.elementwise_mul(x=p_enc, y=U_expr, axis=0) - h_h = layers.elementwise_mul(x=p_enc, y=H_expr, axis=0) - - g = layers.concat(input=[p_enc, U_expr, h_u, h_h], axis=1) - return dropout(g, args) - - -def fusion(g, args): - """Fusion layer""" - m = bi_lstm_encoder( - input_seq=g, gate_size=args.hidden_size, para_name='fusion', args=args) - return dropout(m, args) - - -def lstm_step(x_t, hidden_t_prev, cell_t_prev, size, para_name, args): - """Util function for pointer network""" - - def linear(inputs, para_name, args): - return layers.fc(input=inputs, - size=size, - param_attr=fluid.ParamAttr(name=para_name + '_w'), - bias_attr=fluid.ParamAttr(name=para_name + '_b')) - - input_cat = layers.concat([hidden_t_prev, x_t], axis=1) - forget_gate = layers.sigmoid(x=linear(input_cat, para_name + '_lstm_f', - args)) - input_gate = layers.sigmoid(x=linear(input_cat, para_name + '_lstm_i', - args)) - output_gate = layers.sigmoid(x=linear(input_cat, para_name + '_lstm_o', - args)) - cell_tilde = layers.tanh(x=linear(input_cat, para_name + '_lstm_c', args)) - - cell_t = layers.sums(input=[ - layers.elementwise_mul( - x=forget_gate, y=cell_t_prev), layers.elementwise_mul( - x=input_gate, y=cell_tilde) - ]) - - hidden_t = layers.elementwise_mul(x=output_gate, y=layers.tanh(x=cell_t)) - - return hidden_t, cell_t - - -def point_network_decoder(p_vec, q_vec, hidden_size, args): - """Output layer - pointer network""" - tag = 'pn_decoder:' - init_random = fluid.initializer.Normal(loc=0.0, scale=1.0) - - random_attn = layers.create_parameter( - shape=[1, hidden_size], - dtype='float32', - default_initializer=init_random) - random_attn = layers.fc( - input=random_attn, - size=hidden_size, - act=None, - param_attr=fluid.ParamAttr(name=tag + 'random_attn_fc_w'), - bias_attr=fluid.ParamAttr(name=tag + 'random_attn_fc_b')) - random_attn = layers.reshape(random_attn, shape=[-1]) - U = layers.fc(input=q_vec, - param_attr=fluid.ParamAttr(name=tag + 'q_vec_fc_w'), - bias_attr=False, - size=hidden_size, - act=None) + random_attn - U = layers.tanh(U) - - logits = layers.fc(input=U, - param_attr=fluid.ParamAttr(name=tag + 'logits_fc_w'), - bias_attr=fluid.ParamAttr(name=tag + 'logits_fc_b'), - size=1, - act=None) - scores = layers.sequence_softmax(input=logits) - pooled_vec = layers.elementwise_mul(x=q_vec, y=scores, axis=0) - pooled_vec = layers.sequence_pool(input=pooled_vec, pool_type='sum') - - init_state = layers.fc( - input=pooled_vec, - param_attr=fluid.ParamAttr(name=tag + 'init_state_fc_w'), - bias_attr=fluid.ParamAttr(name=tag + 'init_state_fc_b'), - size=hidden_size, - act=None) - - def custom_dynamic_rnn(p_vec, init_state, hidden_size, para_name, args): - tag = para_name + "custom_dynamic_rnn:" - - def static_rnn(step, - p_vec=p_vec, - init_state=None, - para_name='', - args=args): - tag = para_name + "static_rnn:" - ctx = layers.fc( - input=p_vec, - param_attr=fluid.ParamAttr(name=tag + 'context_fc_w'), - bias_attr=fluid.ParamAttr(name=tag + 'context_fc_b'), - size=hidden_size, - act=None) - - beta = [] - c_prev = init_state - m_prev = init_state - for i in range(step): - m_prev0 = layers.fc( - input=m_prev, - size=hidden_size, - act=None, - param_attr=fluid.ParamAttr(name=tag + 'm_prev0_fc_w'), - bias_attr=fluid.ParamAttr(name=tag + 'm_prev0_fc_b')) - m_prev1 = layers.sequence_expand(x=m_prev0, y=ctx) - - Fk = ctx + m_prev1 - Fk = layers.tanh(Fk) - logits = layers.fc( - input=Fk, - size=1, - act=None, - param_attr=fluid.ParamAttr(name=tag + 'logits_fc_w'), - bias_attr=fluid.ParamAttr(name=tag + 'logits_fc_b')) - - scores = layers.sequence_softmax(input=logits) - attn_ctx = layers.elementwise_mul(x=p_vec, y=scores, axis=0) - attn_ctx = layers.sequence_pool(input=attn_ctx, pool_type='sum') - - hidden_t, cell_t = lstm_step( - attn_ctx, - hidden_t_prev=m_prev, - cell_t_prev=c_prev, - size=hidden_size, - para_name=tag, - args=args) - m_prev = hidden_t - c_prev = cell_t - beta.append(scores) - return beta - - return static_rnn( - 2, p_vec=p_vec, init_state=init_state, para_name=para_name) - - fw_outputs = custom_dynamic_rnn(p_vec, init_state, hidden_size, tag + "fw:", - args) - bw_outputs = custom_dynamic_rnn(p_vec, init_state, hidden_size, tag + "bw:", - args) - - start_prob = layers.elementwise_add( - x=fw_outputs[0], y=bw_outputs[1], axis=0) / 2 - end_prob = layers.elementwise_add( - x=fw_outputs[1], y=bw_outputs[0], axis=0) / 2 - - return start_prob, end_prob - - -def rc_model(hidden_size, vocab, args): - """This function build the whole BiDAF network""" - emb_shape = [vocab.size(), vocab.embed_dim] - start_labels = layers.data( - name="start_lables", shape=[1], dtype='float32', lod_level=1) - end_labels = layers.data( - name="end_lables", shape=[1], dtype='float32', lod_level=1) - - # stage 1:setup input data, embedding table & encode - q_id0 = get_data('q_id0', 1, args) - - q_ids = get_data('q_ids', 2, args) - p_ids_name = 'p_ids' - - p_ids = get_data('p_ids', 2, args) - p_embs = embedding(p_ids, emb_shape, args) - q_embs = embedding(q_ids, emb_shape, args) - drnn = layers.DynamicRNN() - with drnn.block(): - p_emb = drnn.step_input(p_embs) - q_emb = drnn.step_input(q_embs) - - p_enc = encoder(p_emb, 'p_enc', hidden_size, args) - q_enc = encoder(q_emb, 'q_enc', hidden_size, args) - - # stage 2:match - g_i = attn_flow(q_enc, p_enc, p_ids_name, args) - # stage 3:fusion - m_i = fusion(g_i, args) - drnn.output(m_i, q_enc) - - ms, q_encs = drnn() - p_vec = layers.lod_reset(x=ms, y=start_labels) - q_vec = layers.lod_reset(x=q_encs, y=q_id0) - - # stage 4:decode - start_probs, end_probs = point_network_decoder( - p_vec=p_vec, q_vec=q_vec, hidden_size=hidden_size, args=args) - - # calculate model loss - cost0 = layers.sequence_pool( - layers.cross_entropy( - input=start_probs, label=start_labels, soft_label=True), - 'sum') - cost1 = layers.sequence_pool( - layers.cross_entropy( - input=end_probs, label=end_labels, soft_label=True), - 'sum') - - cost0 = layers.mean(cost0) - cost1 = layers.mean(cost1) - cost = cost0 + cost1 - cost.persistable = True - - feeding_list = ["q_ids", "start_lables", "end_lables", "p_ids", "q_id0"] - return cost, start_probs, end_probs, ms, feeding_list diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/run.py b/PaddleNLP/unarchived/machine_reading_comprehension/run.py deleted file mode 100644 index afb2ff8464db99b99132cf4d4e6439e43e7c6e29..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/run.py +++ /dev/null @@ -1,654 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import time -import os -import random -import json -import six -import multiprocessing - -import paddle -import paddle.fluid as fluid -import paddle.fluid.framework as framework -from paddle.fluid.executor import Executor - -import sys -if sys.version[0] == '2': - reload(sys) - sys.setdefaultencoding("utf-8") -sys.path.append('..') - -from args import * -import rc_model -from dataset import BRCDataset -import logging -import pickle -from utils import normalize -from utils import compute_bleu_rouge -from vocab import Vocab - - -def prepare_batch_input(insts, args): - batch_size = len(insts['raw_data']) - inst_num = len(insts['passage_num']) - if batch_size != inst_num: - print("data error %d, %d" % (batch_size, inst_num)) - return None - new_insts = [] - - passage_idx = 0 - for i in range(batch_size): - p_len = 0 - p_id = [] - p_ids = [] - q_ids = [] - q_id = [] - p_id_r = [] - p_ids_r = [] - q_ids_r = [] - q_id_r = [] - - for j in range(insts['passage_num'][i]): - p_ids.append(insts['passage_token_ids'][passage_idx + j]) - p_id = p_id + insts['passage_token_ids'][passage_idx + j] - q_ids.append(insts['question_token_ids'][passage_idx + j]) - q_id = q_id + insts['question_token_ids'][passage_idx + j] - - passage_idx += insts['passage_num'][i] - p_len = len(p_id) - - def _get_label(idx, ref_len): - ret = [0.0] * ref_len - if idx >= 0 and idx < ref_len: - ret[idx] = 1.0 - return [[x] for x in ret] - - start_label = _get_label(insts['start_id'][i], p_len) - end_label = _get_label(insts['end_id'][i], p_len) - new_inst = [q_ids, start_label, end_label, p_ids, q_id] - new_insts.append(new_inst) - return new_insts - - -def batch_reader(batch_list, args): - res = [] - for batch in batch_list: - res.append(prepare_batch_input(batch, args)) - return res - - -def read_multiple(reader, count, clip_last=True): - """ - Stack data from reader for multi-devices. - """ - - def __impl__(): - res = [] - for item in reader(): - res.append(item) - if len(res) == count: - yield res - res = [] - if len(res) == count: - yield res - elif not clip_last: - data = [] - for item in res: - data += item - if len(data) > count: - inst_num_per_part = len(data) // count - yield [ - data[inst_num_per_part * i:inst_num_per_part * (i + 1)] - for i in range(count) - ] - - return __impl__ - - -def LodTensor_Array(lod_tensor): - lod = lod_tensor.lod() - array = np.array(lod_tensor) - new_array = [] - for i in range(len(lod[0]) - 1): - new_array.append(array[lod[0][i]:lod[0][i + 1]]) - return new_array - - -def print_para(train_prog, train_exe, logger, args): - """Print para info for debug purpose""" - if args.para_print: - param_list = train_prog.block(0).all_parameters() - param_name_list = [p.name for p in param_list] - num_sum = 0 - for p_name in param_name_list: - p_array = np.array(train_exe.scope.find_var(p_name).get_tensor()) - param_num = np.prod(p_array.shape) - num_sum = num_sum + param_num - logger.info( - "param: {0}, mean={1} max={2} min={3} num={4} {5}".format( - p_name, - p_array.mean(), - p_array.max(), p_array.min(), p_array.shape, param_num)) - logger.info("total param num: {0}".format(num_sum)) - - -def find_best_answer_for_passage(start_probs, end_probs, passage_len): - """ - Finds the best answer with the maximum start_prob * end_prob from a single passage - """ - if passage_len is None: - passage_len = len(start_probs) - else: - passage_len = min(len(start_probs), passage_len) - best_start, best_end, max_prob = -1, -1, 0 - for start_idx in range(passage_len): - for ans_len in range(args.max_a_len): - end_idx = start_idx + ans_len - if end_idx >= passage_len: - continue - prob = start_probs[start_idx] * end_probs[end_idx] - if prob > max_prob: - best_start = start_idx - best_end = end_idx - max_prob = prob - return (best_start, best_end), max_prob - - -def find_best_answer_for_inst(sample, - start_prob, - end_prob, - inst_lod, - para_prior_scores=(0.44, 0.23, 0.15, 0.09, 0.07)): - """ - Finds the best answer for a sample given start_prob and end_prob for each position. - This will call find_best_answer_for_passage because there are multiple passages in a sample - """ - best_p_idx, best_span, best_score = None, None, 0 - for p_idx, passage in enumerate(sample['passages']): - if p_idx >= args.max_p_num: - continue - if len(start_prob) != len(end_prob): - logger.info('error: {}'.format(sample['question'])) - continue - passage_start = inst_lod[p_idx] - inst_lod[0] - passage_end = inst_lod[p_idx + 1] - inst_lod[0] - passage_len = passage_end - passage_start - passage_len = min(args.max_p_len, len(passage['passage_tokens'])) - answer_span, score = find_best_answer_for_passage( - start_prob[passage_start:passage_end], - end_prob[passage_start:passage_end], passage_len) - if para_prior_scores is not None: - # the Nth prior score = the Number of training samples whose gold answer comes - # from the Nth paragraph / the number of the training samples - score *= para_prior_scores[p_idx] - if score > best_score: - best_score = score - best_p_idx = p_idx - best_span = answer_span - if best_p_idx is None or best_span is None: - best_answer = '' - else: - best_answer = ''.join(sample['passages'][best_p_idx]['passage_tokens'][ - best_span[0]:best_span[1] + 1]) - return best_answer, best_span - - -def validation(inference_program, avg_cost, s_probs, e_probs, match, feed_order, - place, dev_count, vocab, brc_data, logger, args): - """ - do inference with given inference_program - """ - parallel_executor = fluid.ParallelExecutor( - main_program=inference_program, - use_cuda=bool(args.use_gpu), - loss_name=avg_cost.name) - print_para(inference_program, parallel_executor, logger, args) - - # Use test set as validation each pass - total_loss = 0.0 - count = 0 - n_batch_cnt = 0 - n_batch_loss = 0.0 - pred_answers, ref_answers = [], [] - val_feed_list = [ - inference_program.global_block().var(var_name) - for var_name in feed_order - ] - val_feeder = fluid.DataFeeder(val_feed_list, place) - pad_id = vocab.get_id(vocab.pad_token) - dev_reader = lambda:brc_data.gen_mini_batches('dev', args.batch_size, pad_id, shuffle=False) - dev_reader = read_multiple(dev_reader, dev_count) - - for batch_id, batch_list in enumerate(dev_reader(), 1): - feed_data = batch_reader(batch_list, args) - val_fetch_outs = parallel_executor.run( - feed=list(val_feeder.feed_parallel(feed_data, dev_count)), - fetch_list=[avg_cost.name, s_probs.name, e_probs.name, match.name], - return_numpy=False) - total_loss += np.array(val_fetch_outs[0]).sum() - start_probs_m = LodTensor_Array(val_fetch_outs[1]) - end_probs_m = LodTensor_Array(val_fetch_outs[2]) - match_lod = val_fetch_outs[3].lod() - count += len(np.array(val_fetch_outs[0])) - - n_batch_cnt += len(np.array(val_fetch_outs[0])) - n_batch_loss += np.array(val_fetch_outs[0]).sum() - log_every_n_batch = args.log_interval - if log_every_n_batch > 0 and batch_id % log_every_n_batch == 0: - logger.info('Average dev loss from batch {} to {} is {}'.format( - batch_id - log_every_n_batch + 1, batch_id, "%.10f" % ( - n_batch_loss / n_batch_cnt))) - n_batch_loss = 0.0 - n_batch_cnt = 0 - batch_offset = 0 - for idx, batch in enumerate(batch_list): - #one batch - batch_size = len(batch['raw_data']) - batch_range = match_lod[0][batch_offset:batch_offset + batch_size + - 1] - batch_lod = [[batch_range[x], batch_range[x + 1]] - for x in range(len(batch_range[:-1]))] - start_prob_batch = start_probs_m[batch_offset:batch_offset + - batch_size + 1] - end_prob_batch = end_probs_m[batch_offset:batch_offset + batch_size - + 1] - for sample, start_prob_inst, end_prob_inst, inst_range in zip( - batch['raw_data'], start_prob_batch, end_prob_batch, - batch_lod): - #one instance - inst_lod = match_lod[1][inst_range[0]:inst_range[1] + 1] - best_answer, best_span = find_best_answer_for_inst( - sample, start_prob_inst, end_prob_inst, inst_lod) - pred = { - 'question_id': sample['question_id'], - 'question_type': sample['question_type'], - 'answers': [best_answer], - 'entity_answers': [[]], - 'yesno_answers': [] - } - pred_answers.append(pred) - if 'answers' in sample: - ref = { - 'question_id': sample['question_id'], - 'question_type': sample['question_type'], - 'answers': sample['answers'], - 'entity_answers': [[]], - 'yesno_answers': [] - } - ref_answers.append(ref) - batch_offset = batch_offset + batch_size - - result_dir = args.result_dir - result_prefix = args.result_name - if result_dir is not None and result_prefix is not None: - if not os.path.exists(args.result_dir): - os.makedirs(args.result_dir) - result_file = os.path.join(result_dir, result_prefix + '.json') - with open(result_file, 'w') as fout: - for pred_answer in pred_answers: - fout.write(json.dumps(pred_answer, ensure_ascii=False) + '\n') - logger.info('Saving {} results to {}'.format(result_prefix, - result_file)) - - ave_loss = 1.0 * total_loss / count - # compute the bleu and rouge scores if reference answers is provided - if len(ref_answers) > 0: - pred_dict, ref_dict = {}, {} - for pred, ref in zip(pred_answers, ref_answers): - question_id = ref['question_id'] - if len(ref['answers']) > 0: - pred_dict[question_id] = normalize(pred['answers']) - ref_dict[question_id] = normalize(ref['answers']) - bleu_rouge = compute_bleu_rouge(pred_dict, ref_dict) - else: - bleu_rouge = None - return ave_loss, bleu_rouge - - -def l2_loss(train_prog): - param_list = train_prog.block(0).all_parameters() - para_sum = [] - for para in param_list: - para_mul = fluid.layers.elementwise_mul(x=para, y=para, axis=0) - para_sum.append(fluid.layers.reduce_sum(input=para_mul, dim=None)) - return fluid.layers.sums(para_sum) * 0.5 - - -def train(logger, args): - """train a model""" - logger.info('Load data_set and vocab...') - with open(os.path.join(args.vocab_dir, 'vocab.data'), 'rb') as fin: - if six.PY2: - vocab = pickle.load(fin) - else: - vocab = pickle.load(fin, encoding='bytes') - logger.info('vocab size is {} and embed dim is {}'.format(vocab.size( - ), vocab.embed_dim)) - brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len, - args.trainset, args.devset) - logger.info('Converting text into ids...') - brc_data.convert_to_ids(vocab) - logger.info('Initialize the model...') - - if not args.use_gpu: - place = fluid.CPUPlace() - dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - else: - place = fluid.CUDAPlace(0) - dev_count = fluid.core.get_cuda_device_count() - - # build model - main_program = fluid.Program() - startup_prog = fluid.Program() - if args.enable_ce: - main_program.random_seed = args.random_seed - startup_prog.random_seed = args.random_seed - with fluid.program_guard(main_program, startup_prog): - with fluid.unique_name.guard(): - avg_cost, s_probs, e_probs, match, feed_order = rc_model.rc_model( - args.hidden_size, vocab, args) - # clone from default main program and use it as the validation program - inference_program = main_program.clone(for_test=True) - - # build optimizer - if args.optim == 'sgd': - optimizer = fluid.optimizer.SGD( - learning_rate=args.learning_rate) - elif args.optim == 'adam': - optimizer = fluid.optimizer.Adam( - learning_rate=args.learning_rate) - elif args.optim == 'rprop': - optimizer = fluid.optimizer.RMSPropOptimizer( - learning_rate=args.learning_rate) - else: - logger.error('Unsupported optimizer: {}'.format(args.optim)) - exit(-1) - if args.weight_decay > 0.0: - obj_func = avg_cost + args.weight_decay * l2_loss(main_program) - optimizer.minimize(obj_func) - else: - obj_func = avg_cost - optimizer.minimize(obj_func) - - # initialize parameters - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = Executor(place) - if args.load_dir: - logger.info('load from {}'.format(args.load_dir)) - fluid.io.load_persistables( - exe, args.load_dir, main_program=main_program) - else: - exe.run(startup_prog) - embedding_para = fluid.global_scope().find_var( - 'embedding_para').get_tensor() - embedding_para.set(vocab.embeddings.astype(np.float32), place) - - # prepare data - feed_list = [ - main_program.global_block().var(var_name) - for var_name in feed_order - ] - feeder = fluid.DataFeeder(feed_list, place) - - logger.info('Training the model...') - parallel_executor = fluid.ParallelExecutor( - main_program=main_program, - use_cuda=bool(args.use_gpu), - loss_name=avg_cost.name) - print_para(main_program, parallel_executor, logger, args) - - for pass_id in range(1, args.pass_num + 1): - pass_start_time = time.time() - pad_id = vocab.get_id(vocab.pad_token) - if args.enable_ce: - train_reader = lambda:brc_data.gen_mini_batches('train', args.batch_size, pad_id, shuffle=False) - else: - train_reader = lambda:brc_data.gen_mini_batches('train', args.batch_size, pad_id, shuffle=True) - train_reader = read_multiple(train_reader, dev_count) - log_every_n_batch, n_batch_loss = args.log_interval, 0 - total_num, total_loss = 0, 0 - for batch_id, batch_list in enumerate(train_reader(), 1): - feed_data = batch_reader(batch_list, args) - fetch_outs = parallel_executor.run( - feed=list(feeder.feed_parallel(feed_data, dev_count)), - fetch_list=[obj_func.name], - return_numpy=False) - cost_train = np.array(fetch_outs[0]).mean() - total_num += args.batch_size * dev_count - n_batch_loss += cost_train - total_loss += cost_train * args.batch_size * dev_count - - if args.enable_ce and batch_id >= 100: - break - if log_every_n_batch > 0 and batch_id % log_every_n_batch == 0: - print_para(main_program, parallel_executor, logger, - args) - logger.info( - 'Average loss from batch {} to {} is {}'.format( - batch_id - log_every_n_batch + 1, batch_id, - "%.10f" % (n_batch_loss / log_every_n_batch))) - n_batch_loss = 0 - if args.dev_interval > 0 and batch_id % args.dev_interval == 0: - if brc_data.dev_set is not None: - eval_loss, bleu_rouge = validation( - inference_program, avg_cost, s_probs, e_probs, - match, feed_order, place, dev_count, vocab, - brc_data, logger, args) - logger.info('Dev eval loss {}'.format(eval_loss)) - logger.info('Dev eval result: {}'.format( - bleu_rouge)) - pass_end_time = time.time() - time_consumed = pass_end_time - pass_start_time - logger.info('epoch: {0}, epoch_time_cost: {1:.2f}'.format( - pass_id, time_consumed)) - logger.info('Evaluating the model after epoch {}'.format( - pass_id)) - if brc_data.dev_set is not None: - eval_loss, bleu_rouge = validation( - inference_program, avg_cost, s_probs, e_probs, match, - feed_order, place, dev_count, vocab, brc_data, logger, - args) - logger.info('Dev eval loss {}'.format(eval_loss)) - logger.info('Dev eval result: {}'.format(bleu_rouge)) - else: - logger.warning( - 'No dev set is loaded for evaluation in the dataset!') - - logger.info('Average train loss for epoch {} is {}'.format( - pass_id, "%.10f" % (1.0 * total_loss / total_num))) - - if pass_id % args.save_interval == 0: - model_path = os.path.join(args.save_dir, str(pass_id)) - if not os.path.isdir(model_path): - os.makedirs(model_path) - - fluid.io.save_persistables( - executor=exe, - dirname=model_path, - main_program=main_program) - if args.enable_ce: # For CE - print("kpis\ttrain_cost_card%d\t%f" % - (dev_count, total_loss / total_num)) - if brc_data.dev_set is not None: - print("kpis\ttest_cost_card%d\t%f" % - (dev_count, eval_loss)) - print("kpis\ttrain_duration_card%d\t%f" % - (dev_count, time_consumed)) - - -def evaluate(logger, args): - """evaluate a specific model using devset""" - logger.info('Load data_set and vocab...') - with open(os.path.join(args.vocab_dir, 'vocab.data'), 'rb') as fin: - vocab = pickle.load(fin) - logger.info('vocab size is {} and embed dim is {}'.format(vocab.size( - ), vocab.embed_dim)) - brc_data = BRCDataset( - args.max_p_num, args.max_p_len, args.max_q_len, dev_files=args.devset) - logger.info('Converting text into ids...') - brc_data.convert_to_ids(vocab) - logger.info('Initialize the model...') - - # build model - main_program = fluid.Program() - startup_prog = fluid.Program() - with fluid.program_guard(main_program, startup_prog): - with fluid.unique_name.guard(): - avg_cost, s_probs, e_probs, match, feed_order = rc_model.rc_model( - args.hidden_size, vocab, args) - # initialize parameters - if not args.use_gpu: - place = fluid.CPUPlace() - dev_count = int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - else: - place = fluid.CUDAPlace(0) - dev_count = fluid.core.get_cuda_device_count() - - exe = Executor(place) - if args.load_dir: - logger.info('load from {}'.format(args.load_dir)) - fluid.io.load_persistables( - exe, args.load_dir, main_program=main_program) - else: - logger.error('No model file to load ...') - return - - inference_program = main_program.clone(for_test=True) - eval_loss, bleu_rouge = validation( - inference_program, avg_cost, s_probs, e_probs, match, - feed_order, place, dev_count, vocab, brc_data, logger, args) - logger.info('Dev eval loss {}'.format(eval_loss)) - logger.info('Dev eval result: {}'.format(bleu_rouge)) - logger.info('Predicted answers are saved to {}'.format( - os.path.join(args.result_dir))) - - -def predict(logger, args): - """do inference on the test dataset """ - logger.info('Load data_set and vocab...') - with open(os.path.join(args.vocab_dir, 'vocab.data'), 'rb') as fin: - vocab = pickle.load(fin) - logger.info('vocab size is {} and embed dim is {}'.format(vocab.size( - ), vocab.embed_dim)) - brc_data = BRCDataset( - args.max_p_num, args.max_p_len, args.max_q_len, dev_files=args.testset) - logger.info('Converting text into ids...') - brc_data.convert_to_ids(vocab) - logger.info('Initialize the model...') - - # build model - main_program = fluid.Program() - startup_prog = fluid.Program() - with fluid.program_guard(main_program, startup_prog): - with fluid.unique_name.guard(): - avg_cost, s_probs, e_probs, match, feed_order = rc_model.rc_model( - args.hidden_size, vocab, args) - # initialize parameters - if not args.use_gpu: - place = fluid.CPUPlace() - dev_count = int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - else: - place = fluid.CUDAPlace(0) - dev_count = fluid.core.get_cuda_device_count() - - exe = Executor(place) - if args.load_dir: - logger.info('load from {}'.format(args.load_dir)) - fluid.io.load_persistables( - exe, args.load_dir, main_program=main_program) - else: - logger.error('No model file to load ...') - return - - inference_program = main_program.clone(for_test=True) - eval_loss, bleu_rouge = validation( - inference_program, avg_cost, s_probs, e_probs, match, - feed_order, place, dev_count, vocab, brc_data, logger, args) - - -def prepare(logger, args): - """ - checks data, creates the directories, prepare the vocabulary and embeddings - """ - logger.info('Checking the data files...') - for data_path in args.trainset + args.devset + args.testset: - assert os.path.exists(data_path), '{} file does not exist.'.format( - data_path) - logger.info('Preparing the directories...') - for dir_path in [args.vocab_dir, args.save_dir, args.result_dir]: - if not os.path.exists(dir_path): - os.makedirs(dir_path) - - logger.info('Building vocabulary...') - brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len, - args.trainset, args.devset, args.testset) - vocab = Vocab(lower=True) - for word in brc_data.word_iter('train'): - vocab.add(word) - - unfiltered_vocab_size = vocab.size() - vocab.filter_tokens_by_cnt(min_cnt=2) - filtered_num = unfiltered_vocab_size - vocab.size() - logger.info('After filter {} tokens, the final vocab size is {}'.format( - filtered_num, vocab.size())) - - logger.info('Assigning embeddings...') - vocab.randomly_init_embeddings(args.embed_size) - - logger.info('Saving vocab...') - with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout: - pickle.dump(vocab, fout) - - logger.info('Done with preparing!') - - -if __name__ == '__main__': - args = parse_args() - - if args.enable_ce: - random.seed(args.random_seed) - np.random.seed(args.random_seed) - - logger = logging.getLogger("brc") - logger.setLevel(logging.INFO) - formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') - if args.log_path: - file_handler = logging.FileHandler(args.log_path) - file_handler.setLevel(logging.INFO) - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - else: - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.INFO) - console_handler.setFormatter(formatter) - logger.addHandler(console_handler) - args = parse_args() - logger.info('Running with args : {}'.format(args)) - if args.prepare: - prepare(logger, args) - if args.train: - train(logger, args) - if args.evaluate: - evaluate(logger, args) - if args.predict: - predict(logger, args) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/run.sh b/PaddleNLP/unarchived/machine_reading_comprehension/run.sh deleted file mode 100644 index 9f448c0611ae8236dcfd668a093cb3ae72d30603..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/run.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -export CUDA_VISIBLE_DEVICES=1 - -paragraph_extraction () -{ - SOURCE_DIR=$1 - TARGET_DIR=$2 - echo "Start paragraph extraction, this may take a few hours" - echo "Source dir: $SOURCE_DIR" - echo "Target dir: $TARGET_DIR" - mkdir -p $TARGET_DIR/trainset - mkdir -p $TARGET_DIR/devset - mkdir -p $TARGET_DIR/testset - - echo "Processing trainset" - cat $SOURCE_DIR/trainset/search.train.json | python paragraph_extraction.py train \ - > $TARGET_DIR/trainset/search.train.json - cat $SOURCE_DIR/trainset/zhidao.train.json | python paragraph_extraction.py train \ - > $TARGET_DIR/trainset/zhidao.train.json - - echo "Processing devset" - cat $SOURCE_DIR/devset/search.dev.json | python paragraph_extraction.py dev \ - > $TARGET_DIR/devset/search.dev.json - cat $SOURCE_DIR/devset/zhidao.dev.json | python paragraph_extraction.py dev \ - > $TARGET_DIR/devset/zhidao.dev.json - - echo "Processing testset" - cat $SOURCE_DIR/testset/search.test.json | python paragraph_extraction.py test \ - > $TARGET_DIR/testset/search.test.json - cat $SOURCE_DIR/testset/zhidao.test.json | python paragraph_extraction.py test \ - > $TARGET_DIR/testset/zhidao.test.json - echo "Paragraph extraction done!" -} - - -PROCESS_NAME="$1" -case $PROCESS_NAME in - --para_extraction) - # Start paragraph extraction - if [ ! -d data/preprocessed ]; then - echo "Please download the preprocessed data first (See README - Preprocess)" - exit 1 - fi - paragraph_extraction data/preprocessed data/extracted - ;; - --prepare|--train|--evaluate|--predict) - # Start Paddle baseline - python run.py $@ - ;; - *) - echo $"Usage: $0 {--para_extraction|--prepare|--train|--evaluate|--predict}" -esac diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/utils/__init__.py b/PaddleNLP/unarchived/machine_reading_comprehension/utils/__init__.py deleted file mode 100644 index 9d840fd2698d2bb6bccd5bf82cfa51d15e938085..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/utils/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# coding:utf8 -# ============================================================================== -# Copyright 2017 Baidu.com, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -""" -This package implements some utility functions shared by PaddlePaddle -and Tensorflow model implementations. - -Authors: liuyuan(liuyuan04@baidu.com) -Date: 2017/10/06 18:23:06 -""" - -from .dureader_eval import compute_bleu_rouge -from .dureader_eval import normalize -from .preprocess import find_fake_answer -from .preprocess import find_best_question_match - -__all__ = [ - 'compute_bleu_rouge', - 'normalize', - 'find_fake_answer', - 'find_best_question_match', -] diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/utils/download_thirdparty.sh b/PaddleNLP/unarchived/machine_reading_comprehension/utils/download_thirdparty.sh deleted file mode 100755 index cc37da879971c2279edab220485d08d9b20c35fa..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/utils/download_thirdparty.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# ============================================================================== -# Copyright 2017 Baidu.com, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# We use Bleu and Rouge as evaluation metrics, the calculation of these metrics -# relies on the scoring scripts under "https://github.com/tylin/coco-caption" - -bleu_base_url='https://raw.githubusercontent.com/tylin/coco-caption/master/pycocoevalcap/bleu' -bleu_files=("LICENSE" "__init__.py" "bleu.py" "bleu_scorer.py") - -rouge_base_url="https://raw.githubusercontent.com/tylin/coco-caption/master/pycocoevalcap/rouge" -rouge_files=("__init__.py" "rouge.py") - -download() { - local metric=$1; shift; - local base_url=$1; shift; - local fnames=($@); - - mkdir -p ${metric} - for fname in ${fnames[@]}; - do - printf "downloading: %s\n" ${base_url}/${fname} - wget --no-check-certificate ${base_url}/${fname} -O ${metric}/${fname} - done -} - -# prepare rouge -download "rouge_metric" ${rouge_base_url} ${rouge_files[@]} - -# prepare bleu -download "bleu_metric" ${bleu_base_url} ${bleu_files[@]} - -# convert python 2.x source code to python 3.x -2to3 -w "../utils/bleu_metric/bleu_scorer.py" -2to3 -w "../utils/bleu_metric/bleu.py" diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/utils/dureader_eval.py b/PaddleNLP/unarchived/machine_reading_comprehension/utils/dureader_eval.py deleted file mode 100644 index fe43989ca955a01d6cb8b791a085ee33b8fff6c5..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/utils/dureader_eval.py +++ /dev/null @@ -1,545 +0,0 @@ -# -*- coding:utf8 -*- -# ============================================================================== -# Copyright 2017 Baidu.com, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -""" -This module computes evaluation metrics for DuReader dataset. -""" - -import argparse -import json -import sys -import zipfile - -from collections import Counter -from .bleu_metric.bleu import Bleu -from .rouge_metric.rouge import Rouge - -EMPTY = '' -YESNO_LABELS = set(['Yes', 'No', 'Depends']) - - -def normalize(s): - """ - Normalize strings to space joined chars. - - Args: - s: a list of strings. - - Returns: - A list of normalized strings. - """ - if not s: - return s - normalized = [] - for ss in s: - tokens = [c for c in list(ss) if len(c.strip()) != 0] - normalized.append(' '.join(tokens)) - return normalized - - -def data_check(obj, task): - """ - Check data. - - Raises: - Raises AssertionError when data is not legal. - """ - assert 'question_id' in obj, "Missing 'question_id' field." - assert 'question_type' in obj, \ - "Missing 'question_type' field. question_id: {}".format(obj['question_type']) - - assert 'yesno_answers' in obj, \ - "Missing 'yesno_answers' field. question_id: {}".format(obj['question_id']) - assert isinstance(obj['yesno_answers'], list), \ - r"""'yesno_answers' field must be a list, if the 'question_type' is not - 'YES_NO', then this field should be an empty list. - question_id: {}""".format(obj['question_id']) - - assert 'entity_answers' in obj, \ - "Missing 'entity_answers' field. question_id: {}".format(obj['question_id']) - assert isinstance(obj['entity_answers'], list) \ - and len(obj['entity_answers']) > 0, \ - r"""'entity_answers' field must be a list, and has at least one element, - which can be a empty list. question_id: {}""".format(obj['question_id']) - - -def read_file(file_name, task, is_ref=False): - """ - Read predict answers or reference answers from file. - - Args: - file_name: the name of the file containing predict result or reference - result. - - Returns: - A dictionary mapping question_id to the result information. The result - information itself is also a dictionary with has four keys: - - question_type: type of the query. - - yesno_answers: A list of yesno answers corresponding to 'answers'. - - answers: A list of predicted answers. - - entity_answers: A list, each element is also a list containing the entities - tagged out from the corresponding answer string. - """ - - def _open(file_name, mode, zip_obj=None): - if zip_obj is not None: - return zip_obj.open(file_name, mode) - return open(file_name, mode) - - results = {} - keys = ['answers', 'yesno_answers', 'entity_answers', 'question_type'] - if is_ref: - keys += ['source'] - - zf = zipfile.ZipFile(file_name, 'r') if file_name.endswith('.zip') else None - file_list = [file_name] if zf is None else zf.namelist() - - for fn in file_list: - for line in _open(fn, 'r', zip_obj=zf): - try: - obj = json.loads(line.strip()) - except ValueError: - raise ValueError("Every line of data should be legal json") - data_check(obj, task) - qid = obj['question_id'] - assert qid not in results, "Duplicate question_id: {}".format(qid) - results[qid] = {} - for k in keys: - results[qid][k] = obj[k] - return results - - -def compute_bleu_rouge(pred_dict, ref_dict, bleu_order=4): - """ - Compute bleu and rouge scores. - """ - assert set(pred_dict.keys()) == set(ref_dict.keys()), \ - "missing keys: {}".format(set(ref_dict.keys()) - set(pred_dict.keys())) - scores = {} - bleu_scores, _ = Bleu(bleu_order).compute_score(ref_dict, pred_dict) - for i, bleu_score in enumerate(bleu_scores): - scores['Bleu-%d' % (i + 1)] = bleu_score - rouge_score, _ = Rouge().compute_score(ref_dict, pred_dict) - scores['Rouge-L'] = rouge_score - return scores - - -def local_prf(pred_list, ref_list): - """ - Compute local precision recall and f1-score, - given only one prediction list and one reference list - """ - common = Counter(pred_list) & Counter(ref_list) - num_same = sum(common.values()) - if num_same == 0: - return 0, 0, 0 - p = 1.0 * num_same / len(pred_list) - r = 1.0 * num_same / len(ref_list) - f1 = (2 * p * r) / (p + r) - return p, r, f1 - - -def compute_prf(pred_dict, ref_dict): - """ - Compute precision recall and f1-score. - """ - pred_question_ids = set(pred_dict.keys()) - ref_question_ids = set(ref_dict.keys()) - correct_preds, total_correct, total_preds = 0, 0, 0 - for question_id in ref_question_ids: - pred_entity_list = pred_dict.get(question_id, [[]]) - assert len(pred_entity_list) == 1, \ - 'the number of entity list for question_id {} is not 1.'.format(question_id) - pred_entity_list = pred_entity_list[0] - all_ref_entity_lists = ref_dict[question_id] - best_local_f1 = 0 - best_ref_entity_list = None - for ref_entity_list in all_ref_entity_lists: - local_f1 = local_prf(pred_entity_list, ref_entity_list)[2] - if local_f1 > best_local_f1: - best_ref_entity_list = ref_entity_list - best_local_f1 = local_f1 - if best_ref_entity_list is None: - if len(all_ref_entity_lists) > 0: - best_ref_entity_list = sorted( - all_ref_entity_lists, key=lambda x: len(x))[0] - else: - best_ref_entity_list = [] - gold_entities = set(best_ref_entity_list) - pred_entities = set(pred_entity_list) - correct_preds += len(gold_entities & pred_entities) - total_preds += len(pred_entities) - total_correct += len(gold_entities) - p = float(correct_preds) / total_preds if correct_preds > 0 else 0 - r = float(correct_preds) / total_correct if correct_preds > 0 else 0 - f1 = 2 * p * r / (p + r) if correct_preds > 0 else 0 - return {'Precision': p, 'Recall': r, 'F1': f1} - - -def prepare_prf(pred_dict, ref_dict): - """ - Prepares data for calculation of prf scores. - """ - preds = {k: v['entity_answers'] for k, v in pred_dict.items()} - refs = {k: v['entity_answers'] for k, v in ref_dict.items()} - return preds, refs - - -def filter_dict(result_dict, key_tag): - """ - Filter a subset of the result_dict, where keys ends with 'key_tag'. - """ - filtered = {} - for k, v in result_dict.items(): - if k.endswith(key_tag): - filtered[k] = v - return filtered - - -def get_metrics(pred_result, ref_result, task, source): - """ - Computes metrics. - """ - metrics = {} - - ref_result_filtered = {} - pred_result_filtered = {} - if source == 'both': - ref_result_filtered = ref_result - pred_result_filtered = pred_result - else: - for question_id, info in ref_result.items(): - if info['source'] == source: - ref_result_filtered[question_id] = info - if question_id in pred_result: - pred_result_filtered[question_id] = pred_result[question_id] - - if task == 'main' or task == 'all' \ - or task == 'description': - pred_dict, ref_dict = prepare_bleu(pred_result_filtered, - ref_result_filtered, task) - metrics = compute_bleu_rouge(pred_dict, ref_dict) - elif task == 'yesno': - pred_dict, ref_dict = prepare_bleu(pred_result_filtered, - ref_result_filtered, task) - keys = ['Yes', 'No', 'Depends'] - preds = [filter_dict(pred_dict, k) for k in keys] - refs = [filter_dict(ref_dict, k) for k in keys] - - metrics = compute_bleu_rouge(pred_dict, ref_dict) - - for k, pred, ref in zip(keys, preds, refs): - m = compute_bleu_rouge(pred, ref) - k_metric = [(k + '|' + key, v) for key, v in m.items()] - metrics.update(k_metric) - - elif task == 'entity': - pred_dict, ref_dict = prepare_prf(pred_result_filtered, - ref_result_filtered) - pred_dict_bleu, ref_dict_bleu = prepare_bleu(pred_result_filtered, - ref_result_filtered, task) - metrics = compute_prf(pred_dict, ref_dict) - metrics.update(compute_bleu_rouge(pred_dict_bleu, ref_dict_bleu)) - else: - raise ValueError("Illegal task name: {}".format(task)) - - return metrics - - -def prepare_bleu(pred_result, ref_result, task): - """ - Prepares data for calculation of bleu and rouge scores. - """ - pred_list, ref_list = [], [] - qids = ref_result.keys() - for qid in qids: - if task == 'main': - pred, ref = get_main_result(qid, pred_result, ref_result) - elif task == 'yesno': - pred, ref = get_yesno_result(qid, pred_result, ref_result) - elif task == 'all': - pred, ref = get_all_result(qid, pred_result, ref_result) - elif task == 'entity': - pred, ref = get_entity_result(qid, pred_result, ref_result) - elif task == 'description': - pred, ref = get_desc_result(qid, pred_result, ref_result) - else: - raise ValueError("Illegal task name: {}".format(task)) - if pred and ref: - pred_list += pred - ref_list += ref - pred_dict = dict(pred_list) - ref_dict = dict(ref_list) - for qid, ans in ref_dict.items(): - ref_dict[qid] = normalize(ref_dict[qid]) - pred_dict[qid] = normalize(pred_dict.get(qid, [EMPTY])) - if not ans or ans == [EMPTY]: - del ref_dict[qid] - del pred_dict[qid] - - for k, v in pred_dict.items(): - assert len(v) == 1, \ - "There should be only one predict answer. question_id: {}".format(k) - return pred_dict, ref_dict - - -def get_main_result(qid, pred_result, ref_result): - """ - Prepare answers for task 'main'. - - Args: - qid: question_id. - pred_result: A dict include all question_id's result information read - from args.pred_file. - ref_result: A dict incluce all question_id's result information read - from args.ref_file. - Returns: - Two lists, the first one contains predict result, the second - one contains reference result of the same question_id. Each list has - elements of tuple (question_id, answers), 'answers' is a list of strings. - """ - ref_ans = ref_result[qid]['answers'] - if not ref_ans: - ref_ans = [EMPTY] - pred_ans = pred_result.get(qid, {}).get('answers', [])[:1] - if not pred_ans: - pred_ans = [EMPTY] - - return [(qid, pred_ans)], [(qid, ref_ans)] - - -def get_entity_result(qid, pred_result, ref_result): - """ - Prepare answers for task 'entity'. - - Args: - qid: question_id. - pred_result: A dict include all question_id's result information read - from args.pred_file. - ref_result: A dict incluce all question_id's result information read - from args.ref_file. - Returns: - Two lists, the first one contains predict result, the second - one contains reference result of the same question_id. Each list has - elements of tuple (question_id, answers), 'answers' is a list of strings. - """ - if ref_result[qid]['question_type'] != 'ENTITY': - return None, None - return get_main_result(qid, pred_result, ref_result) - - -def get_desc_result(qid, pred_result, ref_result): - """ - Prepare answers for task 'description'. - - Args: - qid: question_id. - pred_result: A dict include all question_id's result information read - from args.pred_file. - ref_result: A dict incluce all question_id's result information read - from args.ref_file. - Returns: - Two lists, the first one contains predict result, the second - one contains reference result of the same question_id. Each list has - elements of tuple (question_id, answers), 'answers' is a list of strings. - """ - if ref_result[qid]['question_type'] != 'DESCRIPTION': - return None, None - return get_main_result(qid, pred_result, ref_result) - - -def get_yesno_result(qid, pred_result, ref_result): - """ - Prepare answers for task 'yesno'. - - Args: - qid: question_id. - pred_result: A dict include all question_id's result information read - from args.pred_file. - ref_result: A dict incluce all question_id's result information read - from args.ref_file. - Returns: - Two lists, the first one contains predict result, the second - one contains reference result of the same question_id. Each list has - elements of tuple (question_id, answers), 'answers' is a list of strings. - """ - - def _uniq(li, is_ref): - uniq_li = [] - left = [] - keys = set() - for k, v in li: - if k not in keys: - uniq_li.append((k, v)) - keys.add(k) - else: - left.append((k, v)) - - if is_ref: - dict_li = dict(uniq_li) - for k, v in left: - dict_li[k] += v - uniq_li = [(k, v) for k, v in dict_li.items()] - return uniq_li - - def _expand_result(uniq_li): - expanded = uniq_li[:] - keys = set([x[0] for x in uniq_li]) - for k in YESNO_LABELS - keys: - expanded.append((k, [EMPTY])) - return expanded - - def _get_yesno_ans(qid, result_dict, is_ref=False): - if qid not in result_dict: - return [(str(qid) + '_' + k, v) for k, v in _expand_result([])] - yesno_answers = result_dict[qid]['yesno_answers'] - answers = result_dict[qid]['answers'] - lbl_ans = _uniq([(k, [v]) for k, v in zip(yesno_answers, answers)], - is_ref) - ret = [(str(qid) + '_' + k, v) for k, v in _expand_result(lbl_ans)] - return ret - - if ref_result[qid]['question_type'] != 'YES_NO': - return None, None - - ref_ans = _get_yesno_ans(qid, ref_result, is_ref=True) - pred_ans = _get_yesno_ans(qid, pred_result) - return pred_ans, ref_ans - - -def get_all_result(qid, pred_result, ref_result): - """ - Prepare answers for task 'all'. - - Args: - qid: question_id. - pred_result: A dict include all question_id's result information read - from args.pred_file. - ref_result: A dict incluce all question_id's result information read - from args.ref_file. - Returns: - Two lists, the first one contains predict result, the second - one contains reference result of the same question_id. Each list has - elements of tuple (question_id, answers), 'answers' is a list of strings. - """ - if ref_result[qid]['question_type'] == 'YES_NO': - return get_yesno_result(qid, pred_result, ref_result) - return get_main_result(qid, pred_result, ref_result) - - -def format_metrics(metrics, task, err_msg): - """ - Format metrics. 'err' field returns any error occured during evaluation. - - Args: - metrics: A dict object contains metrics for different tasks. - task: Task name. - err_msg: Exception raised during evaluation. - Returns: - Formatted result. - """ - result = {} - sources = ["both", "search", "zhidao"] - if err_msg is not None: - return {'errorMsg': str(err_msg), 'errorCode': 1, 'data': []} - data = [] - if task != 'all' and task != 'main': - sources = ["both"] - - if task == 'entity': - metric_names = ["Bleu-4", "Rouge-L"] - metric_names_prf = ["F1", "Precision", "Recall"] - for name in metric_names + metric_names_prf: - for src in sources: - obj = { - "name": name, - "value": round(metrics[src].get(name, 0) * 100, 2), - "type": src, - } - data.append(obj) - elif task == 'yesno': - metric_names = ["Bleu-4", "Rouge-L"] - details = ["Yes", "No", "Depends"] - src = sources[0] - for name in metric_names: - obj = { - "name": name, - "value": round(metrics[src].get(name, 0) * 100, 2), - "type": 'All', - } - data.append(obj) - for d in details: - obj = { - "name": name, - "value": \ - round(metrics[src].get(d + '|' + name, 0) * 100, 2), - "type": d, - } - data.append(obj) - else: - metric_names = ["Bleu-4", "Rouge-L"] - for name in metric_names: - for src in sources: - obj = { - "name": name, - "value": \ - round(metrics[src].get(name, 0) * 100, 2), - "type": src, - } - data.append(obj) - - result["data"] = data - result["errorCode"] = 0 - result["errorMsg"] = "success" - - return result - - -def main(args): - """ - Do evaluation. - """ - err = None - metrics = {} - try: - pred_result = read_file(args.pred_file, args.task) - ref_result = read_file(args.ref_file, args.task, is_ref=True) - sources = ['both', 'search', 'zhidao'] - if args.task not in set(['main', 'all']): - sources = sources[:1] - for source in sources: - metrics[source] = get_metrics(pred_result, ref_result, args.task, - source) - except ValueError as ve: - err = ve - except AssertionError as ae: - err = ae - - print(json.dumps( - format_metrics(metrics, args.task, err), ensure_ascii=False).encode( - 'utf8')) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('pred_file', help='predict file') - parser.add_argument('ref_file', help='reference file') - parser.add_argument( - 'task', help='task name: Main|Yes_No|All|Entity|Description') - - args = parser.parse_args() - args.task = args.task.lower().replace('_', '') - main(args) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/utils/get_vocab.py b/PaddleNLP/unarchived/machine_reading_comprehension/utils/get_vocab.py deleted file mode 100644 index 91de46a1f3f75a64e53d2e44716312fab1bd9323..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/utils/get_vocab.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding:utf8 -*- -# ============================================================================== -# Copyright 2017 Baidu.com, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -""" -Utility function to generate vocabulary file. -""" - -import argparse -import sys -import json - -from itertools import chain - - -def get_vocab(files, vocab_file): - """ - Builds vocabulary file from field 'segmented_paragraphs' - and 'segmented_question'. - - Args: - files: A list of file names. - vocab_file: The file that stores the vocabulary. - """ - vocab = {} - for f in files: - with open(f, 'r') as fin: - for line in fin: - obj = json.loads(line.strip()) - paras = [ - chain(*d['segmented_paragraphs']) for d in obj['documents'] - ] - doc_tokens = chain(*paras) - question_tokens = obj['segmented_question'] - for t in list(doc_tokens) + question_tokens: - vocab[t] = vocab.get(t, 0) + 1 - # output - sorted_vocab = sorted( - [(v, c) for v, c in vocab.items()], key=lambda x: x[1], reverse=True) - with open(vocab_file, 'w') as outf: - for w, c in sorted_vocab: - print >> outf, '{}\t{}'.format(w.encode('utf8'), c) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - '--files', - nargs='+', - required=True, - help='file list to count vocab from.') - parser.add_argument( - '--vocab', required=True, help='file to store counted vocab.') - args = parser.parse_args() - get_vocab(args.files, args.vocab) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/utils/marco_tokenize_data.py b/PaddleNLP/unarchived/machine_reading_comprehension/utils/marco_tokenize_data.py deleted file mode 100644 index 38e56b5e67c54f9867856f7e477697fdbe5d0bd3..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/utils/marco_tokenize_data.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#coding=utf8 - -import os, sys, json -import nltk - - -def _nltk_tokenize(sequence): - tokens = nltk.word_tokenize(sequence) - - cur_char_offset = 0 - token_offsets = [] - token_words = [] - for token in tokens: - cur_char_offset = sequence.find(token, cur_char_offset) - token_offsets.append( - [cur_char_offset, cur_char_offset + len(token) - 1]) - token_words.append(token) - return token_offsets, token_words - - -def segment(input_js): - _, input_js['segmented_question'] = _nltk_tokenize(input_js['question']) - for doc_id, doc in enumerate(input_js['documents']): - doc['segmented_title'] = [] - doc['segmented_paragraphs'] = [] - for para_id, para in enumerate(doc['paragraphs']): - _, seg_para = _nltk_tokenize(para) - doc['segmented_paragraphs'].append(seg_para) - if 'answers' in input_js: - input_js['segmented_answers'] = [] - for answer_id, answer in enumerate(input_js['answers']): - _, seg_answer = _nltk_tokenize(answer) - input_js['segmented_answers'].append(seg_answer) - - -if __name__ == '__main__': - if len(sys.argv) != 2: - print('Usage: tokenize_data.py ') - exit() - - nltk.download('punkt') - - for line in open(sys.argv[1]): - dureader_js = json.loads(line.strip()) - segment(dureader_js) - print(json.dumps(dureader_js)) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/utils/marcov1_to_dureader.py b/PaddleNLP/unarchived/machine_reading_comprehension/utils/marcov1_to_dureader.py deleted file mode 100644 index 833844824eb30a00ee1f6364afe52e66df412bef..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/utils/marcov1_to_dureader.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#coding=utf8 - -import sys -import json -import pandas as pd - - -def trans(input_js): - output_js = {} - output_js['question'] = input_js['query'] - output_js['question_type'] = input_js['query_type'] - output_js['question_id'] = input_js['query_id'] - output_js['fact_or_opinion'] = "" - output_js['documents'] = [] - for para_id, para in enumerate(input_js['passages']): - doc = {} - doc['title'] = "" - if 'is_selected' in para: - doc['is_selected'] = True if para['is_selected'] != 0 else False - doc['paragraphs'] = [para['passage_text']] - output_js['documents'].append(doc) - - if 'answers' in input_js: - output_js['answers'] = input_js['answers'] - return output_js - - -if __name__ == '__main__': - if len(sys.argv) != 2: - print('Usage: marcov1_to_dureader.py ') - exit() - - df = pd.read_json(sys.argv[1]) - for row in df.iterrows(): - marco_js = json.loads(row[1].to_json()) - dureader_js = trans(marco_js) - print(json.dumps(dureader_js)) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/utils/marcov2_to_v1_tojsonl.py b/PaddleNLP/unarchived/machine_reading_comprehension/utils/marcov2_to_v1_tojsonl.py deleted file mode 100644 index 5b102200bc9a03670fb2df19a8023d47f0d3d33d..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/utils/marcov2_to_v1_tojsonl.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import json -import pandas as pd - -if __name__ == '__main__': - if len(sys.argv) != 3: - print('Usage: tojson.py ') - exit() - infile = sys.argv[1] - outfile = sys.argv[2] - df = pd.read_json(infile) - with open(outfile, 'w') as f: - for row in df.iterrows(): - f.write(row[1].to_json() + '\n') diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/utils/preprocess.py b/PaddleNLP/unarchived/machine_reading_comprehension/utils/preprocess.py deleted file mode 100644 index 075d26e45fc5ce116fe12cdc7b958296d24e0f17..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/utils/preprocess.py +++ /dev/null @@ -1,219 +0,0 @@ -############################################################################### -# ============================================================================== -# Copyright 2017 Baidu.com, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -""" -This module finds the most related paragraph of each document according to recall. -""" - -import sys -if sys.version[0] == '2': - reload(sys) - sys.setdefaultencoding("utf-8") -import json -from collections import Counter - - -def precision_recall_f1(prediction, ground_truth): - """ - This function calculates and returns the precision, recall and f1-score - Args: - prediction: prediction string or list to be matched - ground_truth: golden string or list reference - Returns: - floats of (p, r, f1) - Raises: - None - """ - if not isinstance(prediction, list): - prediction_tokens = prediction.split() - else: - prediction_tokens = prediction - if not isinstance(ground_truth, list): - ground_truth_tokens = ground_truth.split() - else: - ground_truth_tokens = ground_truth - common = Counter(prediction_tokens) & Counter(ground_truth_tokens) - num_same = sum(common.values()) - if num_same == 0: - return 0, 0, 0 - p = 1.0 * num_same / len(prediction_tokens) - r = 1.0 * num_same / len(ground_truth_tokens) - f1 = (2 * p * r) / (p + r) - return p, r, f1 - - -def recall(prediction, ground_truth): - """ - This function calculates and returns the recall - Args: - prediction: prediction string or list to be matched - ground_truth: golden string or list reference - Returns: - floats of recall - Raises: - None - """ - return precision_recall_f1(prediction, ground_truth)[1] - - -def f1_score(prediction, ground_truth): - """ - This function calculates and returns the f1-score - Args: - prediction: prediction string or list to be matched - ground_truth: golden string or list reference - Returns: - floats of f1 - Raises: - None - """ - return precision_recall_f1(prediction, ground_truth)[2] - - -def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): - """ - This function calculates and returns the precision, recall and f1-score - Args: - metric_fn: metric function pointer which calculates scores according to corresponding logic. - prediction: prediction string or list to be matched - ground_truth: golden string or list reference - Returns: - floats of (p, r, f1) - Raises: - None - """ - scores_for_ground_truths = [] - for ground_truth in ground_truths: - score = metric_fn(prediction, ground_truth) - scores_for_ground_truths.append(score) - return max(scores_for_ground_truths) - - -def find_best_question_match(doc, question, with_score=False): - """ - For each docment, find the paragraph that matches best to the question. - Args: - doc: The document object. - question: The question tokens. - with_score: If True then the match score will be returned, - otherwise False. - Returns: - The index of the best match paragraph, if with_score=False, - otherwise returns a tuple of the index of the best match paragraph - and the match score of that paragraph. - """ - most_related_para = -1 - max_related_score = 0 - most_related_para_len = 0 - for p_idx, para_tokens in enumerate(doc['segmented_paragraphs']): - if len(question) > 0: - related_score = metric_max_over_ground_truths(recall, para_tokens, - question) - else: - related_score = 0 - - if related_score > max_related_score \ - or (related_score == max_related_score \ - and len(para_tokens) < most_related_para_len): - most_related_para = p_idx - max_related_score = related_score - most_related_para_len = len(para_tokens) - if most_related_para == -1: - most_related_para = 0 - if with_score: - return most_related_para, max_related_score - return most_related_para - - -def find_fake_answer(sample): - """ - For each document, finds the most related paragraph based on recall, - then finds a span that maximize the f1_score compared with the gold answers - and uses this span as a fake answer span - Args: - sample: a sample in the dataset - Returns: - None - Raises: - None - """ - for doc in sample['documents']: - most_related_para = -1 - most_related_para_len = 999999 - max_related_score = 0 - for p_idx, para_tokens in enumerate(doc['segmented_paragraphs']): - if len(sample['segmented_answers']) > 0: - related_score = metric_max_over_ground_truths( - recall, para_tokens, sample['segmented_answers']) - else: - continue - if related_score > max_related_score \ - or (related_score == max_related_score - and len(para_tokens) < most_related_para_len): - most_related_para = p_idx - most_related_para_len = len(para_tokens) - max_related_score = related_score - doc['most_related_para'] = most_related_para - - sample['answer_docs'] = [] - sample['answer_spans'] = [] - sample['fake_answers'] = [] - sample['match_scores'] = [] - - best_match_score = 0 - best_match_d_idx, best_match_span = -1, [-1, -1] - best_fake_answer = None - answer_tokens = set() - for segmented_answer in sample['segmented_answers']: - answer_tokens = answer_tokens | set( - [token for token in segmented_answer]) - for d_idx, doc in enumerate(sample['documents']): - if not doc['is_selected']: - continue - if doc['most_related_para'] == -1: - doc['most_related_para'] = 0 - most_related_para_tokens = doc['segmented_paragraphs'][doc[ - 'most_related_para']][:1000] - for start_tidx in range(len(most_related_para_tokens)): - if most_related_para_tokens[start_tidx] not in answer_tokens: - continue - for end_tidx in range( - len(most_related_para_tokens) - 1, start_tidx - 1, -1): - span_tokens = most_related_para_tokens[start_tidx:end_tidx + 1] - if len(sample['segmented_answers']) > 0: - match_score = metric_max_over_ground_truths( - f1_score, span_tokens, sample['segmented_answers']) - else: - match_score = 0 - if match_score == 0: - break - if match_score > best_match_score: - best_match_d_idx = d_idx - best_match_span = [start_tidx, end_tidx] - best_match_score = match_score - best_fake_answer = ''.join(span_tokens) - if best_match_score > 0: - sample['answer_docs'].append(best_match_d_idx) - sample['answer_spans'].append(best_match_span) - sample['fake_answers'].append(best_fake_answer) - sample['match_scores'].append(best_match_score) - - -if __name__ == '__main__': - for line in sys.stdin: - sample = json.loads(line) - find_fake_answer(sample) - print(json.dumps(sample, encoding='utf8', ensure_ascii=False)) diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/utils/run_marco2dureader_preprocess.sh b/PaddleNLP/unarchived/machine_reading_comprehension/utils/run_marco2dureader_preprocess.sh deleted file mode 100644 index fcb7d67a002ef15384b7c725eb14f3a4dd64ec9e..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/utils/run_marco2dureader_preprocess.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -input_file=$1 -output_file=$2 - -# convert the data from MARCO V2 (json) format to MARCO V1 (jsonl) format. -# the script was forked from MARCO repo. -# the format of MARCO V1 is much more easier to explore. -python3 marcov2_to_v1_tojsonl.py $input_file $input_file.marcov1 - -# convert the data from MARCO V1 format to DuReader format. -python3 marcov1_to_dureader.py $input_file.marcov1 >$input_file.dureader_raw - -# tokenize the data. -python3 marco_tokenize_data.py $input_file.dureader_raw >$input_file.segmented - -# find fake answers (indicating the start and end positions of answers in the document) for train and dev sets. -# note that this should not be applied for test set, since there is no ground truth in test set. -python preprocess.py $input_file.segmented >$output_file - -# remove the temporal data files. -rm -rf $input_file.dureader_raw $input_file.segmented diff --git a/PaddleNLP/unarchived/machine_reading_comprehension/vocab.py b/PaddleNLP/unarchived/machine_reading_comprehension/vocab.py deleted file mode 100644 index c4ebb07ad710b7101858841bd0e9c7a4b90589a7..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/machine_reading_comprehension/vocab.py +++ /dev/null @@ -1,201 +0,0 @@ -# -*- coding:utf8 -*- -# ============================================================================== -# Copyright 2017 Baidu.com, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -""" -This module implements the Vocab class for converting string to id and back -""" - -import numpy as np - - -class Vocab(object): - """ - Implements a vocabulary to store the tokens in the data, with their corresponding embeddings. - """ - - def __init__(self, filename=None, initial_tokens=None, lower=False): - self.id2token = {} - self.token2id = {} - self.token_cnt = {} - self.lower = lower - - self.embed_dim = None - self.embeddings = None - - self.pad_token = '' - self.unk_token = '' - self.split_token = '' - - self.initial_tokens = initial_tokens if initial_tokens is not None else [] - self.initial_tokens.extend( - [self.pad_token, self.unk_token, self.split_token]) - for token in self.initial_tokens: - self.add(token) - - if filename is not None: - self.load_from_file(filename) - - def size(self): - """ - get the size of vocabulary - Returns: - an integer indicating the size - """ - return len(self.id2token) - - def load_from_file(self, file_path): - """ - loads the vocab from file_path - Args: - file_path: a file with a word in each line - """ - for line in open(file_path, 'r'): - token = line.rstrip('\n') - self.add(token) - - def get_id(self, token): - """ - gets the id of a token, returns the id of unk token if token is not in vocab - Args: - key: a string indicating the word - Returns: - an integer - """ - token = token.lower() if self.lower else token - try: - return self.token2id[token] - except KeyError: - return self.token2id[self.unk_token] - - def get_token(self, idx): - """ - gets the token corresponding to idx, returns unk token if idx is not in vocab - Args: - idx: an integer - returns: - a token string - """ - try: - return self.id2token[idx] - except KeyError: - return self.unk_token - - def add(self, token, cnt=1): - """ - adds the token to vocab - Args: - token: a string - cnt: a num indicating the count of the token to add, default is 1 - """ - token = token.lower() if self.lower else token - if token in self.token2id: - idx = self.token2id[token] - else: - idx = len(self.id2token) - self.id2token[idx] = token - self.token2id[token] = idx - if cnt > 0: - if token in self.token_cnt: - self.token_cnt[token] += cnt - else: - self.token_cnt[token] = cnt - return idx - - def filter_tokens_by_cnt(self, min_cnt): - """ - filter the tokens in vocab by their count - Args: - min_cnt: tokens with frequency less than min_cnt is filtered - """ - filtered_tokens = [ - token for token in self.token2id if self.token_cnt[token] >= min_cnt - ] - # rebuild the token x id map - self.token2id = {} - self.id2token = {} - for token in self.initial_tokens: - self.add(token, cnt=0) - for token in filtered_tokens: - self.add(token, cnt=0) - - def randomly_init_embeddings(self, embed_dim): - """ - randomly initializes the embeddings for each token - Args: - embed_dim: the size of the embedding for each token - """ - self.embed_dim = embed_dim - self.embeddings = np.random.rand(self.size(), embed_dim) - for token in [self.pad_token, self.unk_token, self.split_token]: - self.embeddings[self.get_id(token)] = np.zeros([self.embed_dim]) - - def load_pretrained_embeddings(self, embedding_path): - """ - loads the pretrained embeddings from embedding_path, - tokens not in pretrained embeddings will be filtered - Args: - embedding_path: the path of the pretrained embedding file - """ - trained_embeddings = {} - with open(embedding_path, 'r') as fin: - for line in fin: - contents = line.strip().split() - token = contents[0].decode('utf8') - if token not in self.token2id: - continue - trained_embeddings[token] = list(map(float, contents[1:])) - if self.embed_dim is None: - self.embed_dim = len(contents) - 1 - filtered_tokens = trained_embeddings.keys() - # rebuild the token x id map - self.token2id = {} - self.id2token = {} - for token in self.initial_tokens: - self.add(token, cnt=0) - for token in filtered_tokens: - self.add(token, cnt=0) - # load embeddings - self.embeddings = np.zeros([self.size(), self.embed_dim]) - for token in self.token2id.keys(): - if token in trained_embeddings: - self.embeddings[self.get_id(token)] = trained_embeddings[token] - - def convert_to_ids(self, tokens): - """ - Convert a list of tokens to ids, use unk_token if the token is not in vocab. - Args: - tokens: a list of token - Returns: - a list of ids - """ - vec = [self.get_id(label) for label in tokens] - return vec - - def recover_from_ids(self, ids, stop_id=None): - """ - Convert a list of ids to tokens, stop converting if the stop_id is encountered - Args: - ids: a list of ids to convert - stop_id: the stop id, default is None - Returns: - a list of tokens - """ - tokens = [] - for i in ids: - tokens += [self.get_token(i)] - if stop_id is not None and i == stop_id: - break - return tokens diff --git a/PaddleNLP/unarchived/neural_machine_translation/README.md b/PaddleNLP/unarchived/neural_machine_translation/README.md deleted file mode 100644 index a0271ad42e62490282ccc154f6a3c50029b6d13d..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/README.md +++ /dev/null @@ -1,9 +0,0 @@ -The minimum PaddlePaddle version needed for the code sample in this directory is the lastest develop branch. If you are on a version of PaddlePaddle earlier than this, [please update your installation](http://www.paddlepaddle.org/docs/develop/documentation/en/build_and_install/pip_install_en.html). - ---- - -This is a collection of example models for neural machine translation and neural sequence modeling. - -### TODO - -This project is still under active development. diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/.run_ce.sh b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/.run_ce.sh deleted file mode 100755 index fc6638afed895a589201a39e50777a43278742df..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/.run_ce.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - - -DATA_PATH=./data/en-vi/ - -train(){ -python train.py \ - --src_lang en --tar_lang vi \ - --attention True \ - --num_layers 2 \ - --hidden_size 512 \ - --src_vocab_size 17191 \ - --tar_vocab_size 7709 \ - --batch_size 128 \ - --dropout 0.2 \ - --init_scale 0.1 \ - --max_grad_norm 5.0 \ - --train_data_prefix ${DATA_PATH}/train \ - --eval_data_prefix ${DATA_PATH}/tst2012 \ - --test_data_prefix ${DATA_PATH}/tst2013 \ - --vocab_prefix ${DATA_PATH}/vocab \ - --use_gpu True \ - --max_epoch 2 \ - --enable_ce -} - - -cudaid=${transformer:=0} # use 0-th card as default -export CUDA_VISIBLE_DEVICES=$cudaid - -train | python _ce.py - -#cudaid=${transformer_m:=0,1,2,3} # use 0,1,2,3 card as default -#export CUDA_VISIBLE_DEVICES=$cudaid - -#train | python _ce.py diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/README.md b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/README.md deleted file mode 100644 index 60a7331ffcd603784b3b8119580bebb935b7af17..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/README.md +++ /dev/null @@ -1,114 +0,0 @@ -运行本目录下的范例模型需要安装PaddlePaddle Fluid 1.5版。如果您的 PaddlePaddle 安装版本低于此要求，请按照[安装文档](https://www.paddlepaddle.org.cn/#quick-start)中的说明更新 PaddlePaddle 安装版本。 - -# 机器翻译：RNN Search - -以下是本范例模型的简要目录结构及说明： - -```text -. -├── README.md # 文档，本文件 -├── args.py # 训练、预测以及模型参数 -├── train.py # 训练主程序 -├── infer.py # 预测主程序 -├── run.sh # 默认配置的启动脚本 -├── infer.sh # 默认配置的解码脚本 -├── attention_model.py # 带注意力机制的翻译模型配置 -└── base_model.py # 无注意力机制的翻译模型配置 - -``` - -## 简介 -机器翻译（machine translation, MT）是用计算机来实现不同语言之间翻译的技术。被翻译的语言通常称为源语言（source language），翻译成的结果语言称为目标语言（target language）。机器翻译即实现从源语言到目标语言转换的过程，是自然语言处理的重要研究领域之一。 - -近年来，深度学习技术的发展不断为机器翻译任务带来新的突破。直接用神经网络将源语言映射到目标语言，即端到端的神经网络机器翻译（End-to-End Neural Machine Translation, End-to-End NMT）模型逐渐成为主流，此类模型一般简称为NMT模型。 - -本目录包含两个经典的机器翻译模型一个base model（不带attention机制），一个带attention机制的翻译模型 .在现阶段，其表现已被很多新模型（如[Transformer](https://arxiv.org/abs/1706.03762)）超越。但除机器翻译外，该模型是许多序列到序列（sequence to sequence, 以下简称Seq2Seq）类模型的基础，很多解决其他NLP问题的模型均以此模型为基础；因此其在NLP领域具有重要意义，并被广泛用作Baseline. - -本目录下此范例模型的实现，旨在展示如何用Paddle Fluid实现一个带有注意力机制（Attention）的RNN模型来解决Seq2Seq类问题，以及如何使用带有Beam Search算法的解码器。如果您仅仅只是需要在机器翻译方面有着较好翻译效果的模型，则建议您参考[Transformer的Paddle Fluid实现](https://github.com/PaddlePaddle/models/tree/develop/fluid/neural_machine_translation/transformer)。 - -## 模型概览 -RNN Search模型使用了经典的编码器-解码器（Encoder-Decoder）的框架结构来解决Seq2Seq类问题。这种方法先用编码器将源序列编码成vector，再用解码器将该vector解码为目标序列。这其实模拟了人类在进行翻译类任务时的行为：先解析源语言，理解其含义，再根据该含义来写出目标语言的语句。编码器和解码器往往都使用RNN来实现。关于此方法的具体原理和数学表达式，可以参考[深度学习101](http://paddlepaddle.org/documentation/docs/zh/1.2/beginners_guide/basics/machine_translation/index.html). - -本模型中，在编码器方面，我们采用了基于LSTM的多层的encoder；在解码器方面，我们使用了带注意力（Attention）机制的RNN decoder，并同时提供了一个不带注意力机制的解码器实现作为对比；而在预测方面我们使用柱搜索（beam search）算法来生成翻译的目标语句。以下将分别介绍用到的这些方法。 - -## 数据介绍 - -本教程使用[IWSLT'15 English-Vietnamese data ](https://nlp.stanford.edu/projects/nmt/)数据集中的英语到越南语的数据作为训练语料，tst2012的数据作为开发集，tst2013的数据作为测试集 - -### 数据获取 -```sh -cd data && sh download_en-vi.sh -``` - - -## 训练模型 - -`run.sh`包含训练程序的主函数，要使用默认参数开始训练，只需要简单地执行： -```sh -sh run.sh -``` - -```sh - python train.py \ - --src_lang en --tar_lang vi \ - --attention True \ - --num_layers 2 \ - --hidden_size 512 \ - --src_vocab_size 17191 \ - --tar_vocab_size 7709 \ - --batch_size 128 \ - --dropout 0.2 \ - --init_scale 0.1 \ - --max_grad_norm 5.0 \ - --train_data_prefix data/en-vi/train \ - --eval_data_prefix data/en-vi/tst2012 \ - --test_data_prefix data/en-vi/tst2013 \ - --vocab_prefix data/en-vi/vocab \ - --use_gpu True - -``` - - -训练程序会在每个epoch训练结束之后，save一次模型 - -当模型训练完成之后， 可以利用infer.py的脚本进行预测，默认使用beam search的方法进行预测，加载第10个epoch的模型进行预测，对test的数据集进行解码 -```sh -sh infer.sh -``` -如果想预测别的数据文件，只需要将 --infer_file参数进行修改 - -```sh - python infer.py \ - --src_lang en --tar_lang vi \ - --num_layers 2 \ - --hidden_size 512 \ - --src_vocab_size 17191 \ - --tar_vocab_size 7709 \ - --batch_size 128 \ - --dropout 0.2 \ - --init_scale 0.1 \ - --max_grad_norm 5.0 \ - --vocab_prefix data/en-vi/vocab \ - --infer_file data/en-vi/tst2013.en \ - --reload_model model_new/epoch_10/ \ - --use_gpu True - -``` - -## 效果 - -单个模型 beam_size = 10 - -```sh -no attention - -tst2012 BLEU: 11.58 -tst2013 BLEU: 12.20 - - - -with attention - -tst2012 BLEU: 22.21 -tst2013 BLEU: 25.30 -``` diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/__init__.py b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/_ce.py b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/_ce.py deleted file mode 100644 index 934ee7010a2118ec167c2a86639c4cfbf9e06a48..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/_ce.py +++ /dev/null @@ -1,62 +0,0 @@ -####this file is only used for continuous evaluation test! - -import os -import sys -sys.path.insert(0, os.environ['ceroot']) -from kpi import CostKpi, DurationKpi, AccKpi - -#### NOTE kpi.py should shared in models in some way!!!! - -train_ppl_card1_kpi = CostKpi('train_ppl_card1', 0.02, 0, actived=True) -train_duration_card1_kpi = DurationKpi( - 'train_duration_card1', 0.02, 0, actived=True) -#train_ppl_card4_kpi = CostKpi('train_ppl_card4', 0.02, 0, actived=True) -#train_duration_card4_kpi = DurationKpi( -# 'train_duration_card4', 0.02, 0, actived=True) - -tracking_kpis = [ - train_ppl_card1_kpi, - train_duration_card1_kpi, -] - - -def parse_log(log): - ''' - This method should be implemented by model developers. - The suggestion: - each line in the log should be key, value, for example: - " - train_cost\t1.0 - test_cost\t1.0 - train_cost\t1.0 - train_cost\t1.0 - train_acc\t1.2 - " - ''' - for line in log.split('\n'): - fs = line.strip().split('\t') - print(fs) - if len(fs) == 3 and fs[0] == 'kpis': - print("-----%s" % fs) - kpi_name = fs[1] - kpi_value = float(fs[2]) - yield kpi_name, kpi_value - - -def log_to_ce(log): - kpi_tracker = {} - for kpi in tracking_kpis: - kpi_tracker[kpi.name] = kpi - - for (kpi_name, kpi_value) in parse_log(log): - print(kpi_name, kpi_value) - kpi_tracker[kpi_name].add_record(kpi_value) - kpi_tracker[kpi_name].persist() - - -if __name__ == '__main__': - log = sys.stdin.read() - print("*****") - print(log) - print("****") - log_to_ce(log) diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/args.py b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/args.py deleted file mode 100644 index 8d27f9abf05d34d4b94b0a03f13a31eee661311c..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/args.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import distutils.util - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--train_data_prefix", type=str, help="file prefix for train data") - parser.add_argument( - "--eval_data_prefix", type=str, help="file prefix for eval data") - parser.add_argument( - "--test_data_prefix", type=str, help="file prefix for test data") - parser.add_argument( - "--vocab_prefix", type=str, help="file prefix for vocab") - parser.add_argument("--src_lang", type=str, help="source language suffix") - parser.add_argument("--tar_lang", type=str, help="target language suffix") - - parser.add_argument( - "--attention", - type=bool, - default=False, - help="Whether use attention model") - - parser.add_argument( - "--optimizer", - type=str, - default='adam', - help="optimizer to use, only supprt[sgd|adam]") - - parser.add_argument( - "--learning_rate", - type=float, - default=0.001, - help="learning rate for optimizer") - - parser.add_argument( - "--num_layers", - type=int, - default=1, - help="layers number of encoder and decoder") - parser.add_argument( - "--hidden_size", - type=int, - default=100, - help="hidden size of encoder and decoder") - parser.add_argument("--src_vocab_size", type=int, help="source vocab size") - parser.add_argument("--tar_vocab_size", type=int, help="target vocab size") - - parser.add_argument( - "--batch_size", type=int, help="batch size of each step") - - parser.add_argument( - "--max_epoch", type=int, default=12, help="max epoch for the training") - - parser.add_argument( - "--max_len", - type=int, - default=50, - help="max length for source and target sentence") - parser.add_argument( - "--dropout", type=float, default=0.0, help="drop probability") - parser.add_argument( - "--init_scale", - type=float, - default=0.0, - help="init scale for parameter") - parser.add_argument( - "--max_grad_norm", - type=float, - default=5.0, - help="max grad norm for global norm clip") - - parser.add_argument( - "--model_path", - type=str, - default='./model', - help="model path for model to save") - - parser.add_argument( - "--reload_model", type=str, help="reload model to inference") - - parser.add_argument( - "--infer_file", type=str, help="file name for inference") - parser.add_argument( - "--infer_output_file", - type=str, - default='./infer_output', - help="file name for inference output") - parser.add_argument( - "--beam_size", type=int, default=10, help="file name for inference") - - parser.add_argument( - '--use_gpu', - type=bool, - default=False, - help='Whether using gpu [True|False]') - - parser.add_argument( - "--enable_ce", - action='store_true', - help="The flag indicating whether to run the task " - "for continuous evaluation.") - - parser.add_argument( - "--parallel", - action='store_true', - help="Whether execute with the data_parallel mode.") - - parser.add_argument( - "--profile", - action='store_true', - help="Whether enable the profile.") - - args = parser.parse_args() - return args diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/attention_model.py b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/attention_model.py deleted file mode 100644 index aef110f5ee25416d5b4113f47ac838c22ebfc863..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/attention_model.py +++ /dev/null @@ -1,484 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import paddle.fluid.layers as layers -import paddle.fluid as fluid -from paddle.fluid.layers.control_flow import StaticRNN -import numpy as np -from paddle.fluid import ParamAttr -from paddle.fluid.contrib.layers import basic_lstm, BasicLSTMUnit -from base_model import BaseModel - -INF = 1. * 1e5 -alpha = 0.6 - - -class AttentionModel(BaseModel): - def __init__(self, - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=1, - init_scale=0.1, - dropout=None, - batch_first=True): - super(AttentionModel, self).__init__( - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout, - batch_first=batch_first) - - def _build_decoder(self, - enc_last_hidden, - enc_last_cell, - mode='train', - beam_size=10): - - dec_input = layers.transpose(self.tar_emb, [1, 0, 2]) - dec_unit_list = [] - for i in range(self.num_layers): - new_name = "dec_layers_" + str(i) - dec_unit_list.append( - BasicLSTMUnit( - new_name, - self.hidden_size, - ParamAttr(initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)), - ParamAttr(initializer=fluid.initializer.Constant(0.0)), )) - - - attention_weight = layers.create_parameter([self.hidden_size * 2, self.hidden_size], dtype="float32", name="attention_weight", \ - default_initializer=fluid.initializer.UniformInitializer(low=-self.init_scale, high=self.init_scale)) - - memory_weight = layers.create_parameter([self.hidden_size, self.hidden_size], dtype="float32", name="memory_weight", \ - default_initializer=fluid.initializer.UniformInitializer(low=-self.init_scale, high=self.init_scale)) - - def dot_attention(query, memory, mask=None): - attn = layers.matmul(query, memory, transpose_y=True) - - if mask: - attn = layers.transpose(attn, [1, 0, 2]) - attn = layers.elementwise_add(attn, mask * 1000000000, -1) - attn = layers.transpose(attn, [1, 0, 2]) - weight = layers.softmax(attn) - weight_memory = layers.matmul(weight, memory) - - return weight_memory, weight - - max_src_seq_len = layers.shape(self.src)[1] - src_mask = layers.sequence_mask( - self.src_sequence_length, maxlen=max_src_seq_len, dtype='float32') - - softmax_weight = layers.create_parameter([self.hidden_size, self.tar_vocab_size], dtype="float32", name="softmax_weight", \ - default_initializer=fluid.initializer.UniformInitializer(low=-self.init_scale, high=self.init_scale)) - - def decoder_step(currrent_in, pre_feed, pre_hidden_array, - pre_cell_array, enc_memory): - new_hidden_array = [] - new_cell_array = [] - - step_input = layers.concat([currrent_in, pre_feed], 1) - - for i in range(self.num_layers): - pre_hidden = pre_hidden_array[i] - pre_cell = pre_cell_array[i] - - new_hidden, new_cell = dec_unit_list[i](step_input, pre_hidden, - pre_cell) - - new_hidden_array.append(new_hidden) - new_cell_array.append(new_cell) - - step_input = new_hidden - - memory_mask = src_mask - 1.0 - enc_memory = layers.matmul(enc_memory, memory_weight) - att_in = layers.unsqueeze(step_input, [1]) - dec_att, _ = dot_attention(att_in, enc_memory) - dec_att = layers.squeeze(dec_att, [1]) - concat_att_out = layers.concat([dec_att, step_input], 1) - concat_att_out = layers.matmul(concat_att_out, attention_weight) - - return concat_att_out, new_hidden_array, new_cell_array - - if mode == "train": - dec_rnn = StaticRNN() - with dec_rnn.step(): - step_input = dec_rnn.step_input(dec_input) - input_feed = dec_rnn.memory( - batch_ref=dec_input, shape=[-1, self.hidden_size]) - step_input = layers.concat([step_input, input_feed], 1) - - for i in range(self.num_layers): - pre_hidden = dec_rnn.memory(init=enc_last_hidden[i]) - pre_cell = dec_rnn.memory(init=enc_last_cell[i]) - - new_hidden, new_cell = dec_unit_list[i]( - step_input, pre_hidden, pre_cell) - - dec_rnn.update_memory(pre_hidden, new_hidden) - dec_rnn.update_memory(pre_cell, new_cell) - - step_input = new_hidden - - if self.dropout != None and self.dropout > 0.0: - print("using dropout", self.dropout) - step_input = fluid.layers.dropout( - step_input, - dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') - memory_mask = src_mask - 1.0 - enc_memory = layers.matmul(self.enc_output, memory_weight) - att_in = layers.unsqueeze(step_input, [1]) - dec_att, _ = dot_attention(att_in, enc_memory, memory_mask) - dec_att = layers.squeeze(dec_att, [1]) - concat_att_out = layers.concat([dec_att, step_input], 1) - concat_att_out = layers.matmul(concat_att_out, attention_weight) - #concat_att_out = layers.tanh( concat_att_out ) - - dec_rnn.update_memory(input_feed, concat_att_out) - - dec_rnn.step_output(concat_att_out) - - dec_rnn_out = dec_rnn() - dec_output = layers.transpose(dec_rnn_out, [1, 0, 2]) - - dec_output = layers.matmul(dec_output, softmax_weight) - - return dec_output - elif mode == 'beam_search': - - max_length = max_src_seq_len * 2 - #max_length = layers.fill_constant( [1], dtype='int32', value = 10) - pre_ids = layers.fill_constant([1, 1], dtype='int64', value=1) - full_ids = layers.fill_constant([1, 1], dtype='int64', value=1) - - score = layers.fill_constant([1], dtype='float32', value=0.0) - - #eos_ids = layers.fill_constant( [1, 1], dtype='int64', value=2) - - pre_hidden_array = [] - pre_cell_array = [] - pre_feed = layers.fill_constant( - [beam_size, self.hidden_size], dtype='float32', value=0) - for i in range(self.num_layers): - pre_hidden_array.append( - layers.expand(enc_last_hidden[i], [beam_size, 1])) - pre_cell_array.append( - layers.expand(enc_last_cell[i], [beam_size, 1])) - - eos_ids = layers.fill_constant([beam_size], dtype='int64', value=2) - init_score = np.zeros((beam_size)).astype('float32') - init_score[1:] = -INF - pre_score = layers.assign(init_score) - #pre_score = layers.fill_constant( [1,], dtype='float32', value= 0.0) - tokens = layers.fill_constant( - [beam_size, 1], dtype='int64', value=1) - - enc_memory = layers.expand(self.enc_output, [beam_size, 1, 1]) - - pre_tokens = layers.fill_constant( - [beam_size, 1], dtype='int64', value=1) - - finished_seq = layers.fill_constant( - [beam_size, 1], dtype='int64', value=0) - finished_scores = layers.fill_constant( - [beam_size], dtype='float32', value=-INF) - finished_flag = layers.fill_constant( - [beam_size], dtype='float32', value=0.0) - - step_idx = layers.fill_constant(shape=[1], dtype='int32', value=0) - cond = layers.less_than( - x=step_idx, y=max_length) # default force_cpu=True - - parent_idx = layers.fill_constant([1], dtype='int32', value=0) - while_op = layers.While(cond) - - def compute_topk_scores_and_seq(sequences, - scores, - scores_to_gather, - flags, - beam_size, - select_beam=None, - generate_id=None): - scores = layers.reshape(scores, shape=[1, -1]) - _, topk_indexs = layers.topk(scores, k=beam_size) - - topk_indexs = layers.reshape(topk_indexs, shape=[-1]) - - # gather result - - top_seq = layers.gather(sequences, topk_indexs) - topk_flags = layers.gather(flags, topk_indexs) - topk_gather_scores = layers.gather(scores_to_gather, - topk_indexs) - - if select_beam: - topk_beam = layers.gather(select_beam, topk_indexs) - else: - topk_beam = select_beam - - if generate_id: - topk_id = layers.gather(generate_id, topk_indexs) - else: - topk_id = generate_id - return top_seq, topk_gather_scores, topk_flags, topk_beam, topk_id - - def grow_alive(curr_seq, curr_scores, curr_log_probs, curr_finished, - select_beam, generate_id): - curr_scores += curr_finished * -INF - return compute_topk_scores_and_seq( - curr_seq, - curr_scores, - curr_log_probs, - curr_finished, - beam_size, - select_beam, - generate_id=generate_id) - - def grow_finished(finished_seq, finished_scores, finished_flag, - curr_seq, curr_scores, curr_finished): - finished_seq = layers.concat( - [ - finished_seq, layers.fill_constant( - [beam_size, 1], dtype='int64', value=1) - ], - axis=1) - curr_scores += (1.0 - curr_finished) * -INF - #layers.Print( curr_scores, message="curr scores") - curr_finished_seq = layers.concat( - [finished_seq, curr_seq], axis=0) - curr_finished_scores = layers.concat( - [finished_scores, curr_scores], axis=0) - curr_finished_flags = layers.concat( - [finished_flag, curr_finished], axis=0) - - return compute_topk_scores_and_seq( - curr_finished_seq, curr_finished_scores, - curr_finished_scores, curr_finished_flags, beam_size) - - def is_finished(alive_log_prob, finished_scores, - finished_in_finished): - - max_out_len = 200 - max_length_penalty = layers.pow(layers.fill_constant( - [1], dtype='float32', value=((5.0 + max_out_len) / 6.0)), - alpha) - - lower_bound_alive_score = layers.slice( - alive_log_prob, starts=[0], ends=[1], - axes=[0]) / max_length_penalty - - lowest_score_of_fininshed_in_finished = finished_scores * finished_in_finished - lowest_score_of_fininshed_in_finished += ( - 1.0 - finished_in_finished) * -INF - lowest_score_of_fininshed_in_finished = layers.reduce_min( - lowest_score_of_fininshed_in_finished) - - met = layers.less_than(lower_bound_alive_score, - lowest_score_of_fininshed_in_finished) - met = layers.cast(met, 'float32') - bound_is_met = layers.reduce_sum(met) - - finished_eos_num = layers.reduce_sum(finished_in_finished) - - finish_cond = layers.less_than( - finished_eos_num, - layers.fill_constant( - [1], dtype='float32', value=beam_size)) - - return finish_cond - - def grow_top_k(step_idx, alive_seq, alive_log_prob, parant_idx): - pre_ids = alive_seq - - dec_step_emb = layers.embedding( - input=pre_ids, - size=[self.tar_vocab_size, self.hidden_size], - dtype='float32', - is_sparse=False, - param_attr=fluid.ParamAttr( - name='target_embedding', - initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale))) - - dec_att_out, new_hidden_array, new_cell_array = decoder_step( - dec_step_emb, pre_feed, pre_hidden_array, pre_cell_array, - enc_memory) - - projection = layers.matmul(dec_att_out, softmax_weight) - - logits = layers.softmax(projection) - current_log = layers.elementwise_add( - x=layers.log(logits), y=alive_log_prob, axis=0) - base_1 = layers.cast(step_idx, 'float32') + 6.0 - base_1 /= 6.0 - length_penalty = layers.pow(base_1, alpha) - - len_pen = layers.pow(( - (5. + layers.cast(step_idx + 1, 'float32')) / 6.), alpha) - - current_log = layers.reshape(current_log, shape=[1, -1]) - - current_log = current_log / length_penalty - topk_scores, topk_indices = layers.topk( - input=current_log, k=beam_size) - - topk_scores = layers.reshape(topk_scores, shape=[-1]) - - topk_log_probs = topk_scores * length_penalty - - generate_id = layers.reshape( - topk_indices, shape=[-1]) % self.tar_vocab_size - - selected_beam = layers.reshape( - topk_indices, shape=[-1]) // self.tar_vocab_size - - topk_finished = layers.equal(generate_id, eos_ids) - - topk_finished = layers.cast(topk_finished, 'float32') - - generate_id = layers.reshape(generate_id, shape=[-1, 1]) - - pre_tokens_list = layers.gather(tokens, selected_beam) - - full_tokens_list = layers.concat( - [pre_tokens_list, generate_id], axis=1) - - - return full_tokens_list, topk_log_probs, topk_scores, topk_finished, selected_beam, generate_id, \ - dec_att_out, new_hidden_array, new_cell_array - - with while_op.block(): - topk_seq, topk_log_probs, topk_scores, topk_finished, topk_beam, topk_generate_id, attention_out, new_hidden_array, new_cell_array = \ - grow_top_k( step_idx, pre_tokens, pre_score, parent_idx) - alive_seq, alive_log_prob, _, alive_beam, alive_id = grow_alive( - topk_seq, topk_scores, topk_log_probs, topk_finished, - topk_beam, topk_generate_id) - - finished_seq_2, finished_scores_2, finished_flags_2, _, _ = grow_finished( - finished_seq, finished_scores, finished_flag, topk_seq, - topk_scores, topk_finished) - - finished_cond = is_finished(alive_log_prob, finished_scores_2, - finished_flags_2) - - layers.increment(x=step_idx, value=1.0, in_place=True) - - layers.assign(alive_beam, parent_idx) - layers.assign(alive_id, pre_tokens) - layers.assign(alive_log_prob, pre_score) - layers.assign(alive_seq, tokens) - layers.assign(finished_seq_2, finished_seq) - layers.assign(finished_scores_2, finished_scores) - layers.assign(finished_flags_2, finished_flag) - - # update init_hidden, init_cell, input_feed - new_feed = layers.gather(attention_out, parent_idx) - layers.assign(new_feed, pre_feed) - for i in range(self.num_layers): - new_hidden_var = layers.gather(new_hidden_array[i], - parent_idx) - layers.assign(new_hidden_var, pre_hidden_array[i]) - new_cell_var = layers.gather(new_cell_array[i], parent_idx) - layers.assign(new_cell_var, pre_cell_array[i]) - - length_cond = layers.less_than(x=step_idx, y=max_length) - layers.logical_and(x=length_cond, y=finished_cond, out=cond) - - tokens_with_eos = tokens - - all_seq = layers.concat([tokens_with_eos, finished_seq], axis=0) - all_score = layers.concat([pre_score, finished_scores], axis=0) - _, topk_index = layers.topk(all_score, k=beam_size) - topk_index = layers.reshape(topk_index, shape=[-1]) - final_seq = layers.gather(all_seq, topk_index) - final_score = layers.gather(all_score, topk_index) - - return final_seq - elif mode == 'greedy_search': - max_length = max_src_seq_len * 2 - #max_length = layers.fill_constant( [1], dtype='int32', value = 10) - pre_ids = layers.fill_constant([1, 1], dtype='int64', value=1) - full_ids = layers.fill_constant([1, 1], dtype='int64', value=1) - - score = layers.fill_constant([1], dtype='float32', value=0.0) - - eos_ids = layers.fill_constant([1, 1], dtype='int64', value=2) - - pre_hidden_array = [] - pre_cell_array = [] - pre_feed = layers.fill_constant( - [1, self.hidden_size], dtype='float32', value=0) - for i in range(self.num_layers): - pre_hidden_array.append(enc_last_hidden[i]) - pre_cell_array.append(enc_last_cell[i]) - #pre_hidden_array.append( layers.fill_constant( [1, hidden_size], dtype='float32', value=0) ) - #pre_cell_array.append( layers.fill_constant( [1, hidden_size], dtype='float32', value=0) ) - - step_idx = layers.fill_constant(shape=[1], dtype='int32', value=0) - cond = layers.less_than( - x=step_idx, y=max_length) # default force_cpu=True - while_op = layers.While(cond) - - with while_op.block(): - - dec_step_emb = layers.embedding( - input=pre_ids, - size=[self.tar_vocab_size, self.hidden_size], - dtype='float32', - is_sparse=False, - param_attr=fluid.ParamAttr( - name='target_embedding', - initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale))) - - dec_att_out, new_hidden_array, new_cell_array = decoder_step( - dec_step_emb, pre_feed, pre_hidden_array, pre_cell_array, - self.enc_output) - - projection = layers.matmul(dec_att_out, softmax_weight) - - logits = layers.softmax(projection) - logits = layers.log(logits) - - current_log = layers.elementwise_add(logits, score, axis=0) - - topk_score, topk_indices = layers.topk(input=current_log, k=1) - - new_ids = layers.concat([full_ids, topk_indices]) - layers.assign(new_ids, full_ids) - #layers.Print( full_ids, message="ful ids") - layers.assign(topk_score, score) - layers.assign(topk_indices, pre_ids) - layers.assign(dec_att_out, pre_feed) - for i in range(self.num_layers): - layers.assign(new_hidden_array[i], pre_hidden_array[i]) - layers.assign(new_cell_array[i], pre_cell_array[i]) - - layers.increment(x=step_idx, value=1.0, in_place=True) - - eos_met = layers.not_equal(topk_indices, eos_ids) - length_cond = layers.less_than(x=step_idx, y=max_length) - layers.logical_and(x=length_cond, y=eos_met, out=cond) - - return full_ids diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/base_model.py b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/base_model.py deleted file mode 100644 index bebfc2f86ccf46e61639ac4bb723a9de8b08a0eb..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/base_model.py +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import paddle.fluid.layers as layers -import paddle.fluid as fluid -from paddle.fluid.layers.control_flow import StaticRNN as PaddingRNN -import numpy as np -from paddle.fluid import ParamAttr -from paddle.fluid.contrib.layers import basic_lstm, BasicLSTMUnit - -INF = 1. * 1e5 -alpha = 0.6 - - -class BaseModel(object): - def __init__(self, - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=1, - init_scale=0.1, - dropout=None, - batch_first=True): - - self.hidden_size = hidden_size - self.src_vocab_size = src_vocab_size - self.tar_vocab_size = tar_vocab_size - self.batch_size = batch_size - self.num_layers = num_layers - self.init_scale = init_scale - self.dropout = dropout - self.batch_first = batch_first - - def _build_data(self): - self.src = layers.data(name="src", shape=[-1, 1, 1], dtype='int64') - self.src_sequence_length = layers.data( - name="src_sequence_length", shape=[-1], dtype='int32') - - self.tar = layers.data(name="tar", shape=[-1, 1, 1], dtype='int64') - self.tar_sequence_length = layers.data( - name="tar_sequence_length", shape=[-1], dtype='int32') - self.label = layers.data(name="label", shape=[-1, 1, 1], dtype='int64') - - def _emebdding(self): - self.src_emb = layers.embedding( - input=self.src, - size=[self.src_vocab_size, self.hidden_size], - dtype='float32', - is_sparse=False, - param_attr=fluid.ParamAttr( - name='source_embedding', - initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale))) - self.tar_emb = layers.embedding( - input=self.tar, - size=[self.tar_vocab_size, self.hidden_size], - dtype='float32', - is_sparse=False, - param_attr=fluid.ParamAttr( - name='target_embedding', - initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale))) - - def _build_encoder(self): - self.enc_output, enc_last_hidden, enc_last_cell = basic_lstm( self.src_emb, None, None, self.hidden_size, num_layers=self.num_layers, batch_first=self.batch_first, \ - dropout_prob=self.dropout, \ - param_attr = ParamAttr( initializer=fluid.initializer.UniformInitializer(low=-self.init_scale, high=self.init_scale) ), \ - bias_attr = ParamAttr( initializer = fluid.initializer.Constant(0.0) ), \ - sequence_length=self.src_sequence_length) - - return self.enc_output, enc_last_hidden, enc_last_cell - - def _build_decoder(self, - enc_last_hidden, - enc_last_cell, - mode='train', - beam_size=10): - softmax_weight = layers.create_parameter([self.hidden_size, self.tar_vocab_size], dtype="float32", name="softmax_weight", \ - default_initializer=fluid.initializer.UniformInitializer(low=-self.init_scale, high=self.init_scale)) - if mode == 'train': - dec_output, dec_last_hidden, dec_last_cell = basic_lstm( self.tar_emb, enc_last_hidden, enc_last_cell, \ - self.hidden_size, num_layers=self.num_layers, \ - batch_first=self.batch_first, \ - dropout_prob=self.dropout, \ - param_attr = ParamAttr( initializer=fluid.initializer.UniformInitializer(low=-self.init_scale, high=self.init_scale) ), \ - bias_attr = ParamAttr( initializer = fluid.initializer.Constant(0.0) )) - - dec_output = layers.matmul(dec_output, softmax_weight) - - return dec_output - elif mode == 'beam_search' or mode == 'greedy_search': - dec_unit_list = [] - name = 'basic_lstm' - for i in range(self.num_layers): - new_name = name + "_layers_" + str(i) - dec_unit_list.append( - BasicLSTMUnit( - new_name, self.hidden_size, dtype='float32')) - - def decoder_step(current_in, pre_hidden_array, pre_cell_array): - new_hidden_array = [] - new_cell_array = [] - - step_in = current_in - for i in range(self.num_layers): - pre_hidden = pre_hidden_array[i] - pre_cell = pre_cell_array[i] - - new_hidden, new_cell = dec_unit_list[i](step_in, pre_hidden, - pre_cell) - - new_hidden_array.append(new_hidden) - new_cell_array.append(new_cell) - - step_in = new_hidden - - return step_in, new_hidden_array, new_cell_array - - if mode == 'beam_search': - max_src_seq_len = layers.shape(self.src)[1] - max_length = max_src_seq_len * 2 - #max_length = layers.fill_constant( [1], dtype='int32', value = 10) - pre_ids = layers.fill_constant([1, 1], dtype='int64', value=1) - full_ids = layers.fill_constant([1, 1], dtype='int64', value=1) - - score = layers.fill_constant([1], dtype='float32', value=0.0) - - #eos_ids = layers.fill_constant( [1, 1], dtype='int64', value=2) - - pre_hidden_array = [] - pre_cell_array = [] - pre_feed = layers.fill_constant( - [beam_size, self.hidden_size], dtype='float32', value=0) - for i in range(self.num_layers): - pre_hidden_array.append( - layers.expand(enc_last_hidden[i], [beam_size, 1])) - pre_cell_array.append( - layers.expand(enc_last_cell[i], [beam_size, 1])) - - eos_ids = layers.fill_constant( - [beam_size], dtype='int64', value=2) - init_score = np.zeros((beam_size)).astype('float32') - init_score[1:] = -INF - pre_score = layers.assign(init_score) - #pre_score = layers.fill_constant( [1,], dtype='float32', value= 0.0) - tokens = layers.fill_constant( - [beam_size, 1], dtype='int64', value=1) - - enc_memory = layers.expand(self.enc_output, [beam_size, 1, 1]) - - pre_tokens = layers.fill_constant( - [beam_size, 1], dtype='int64', value=1) - - finished_seq = layers.fill_constant( - [beam_size, 1], dtype='int64', value=0) - finished_scores = layers.fill_constant( - [beam_size], dtype='float32', value=-INF) - finished_flag = layers.fill_constant( - [beam_size], dtype='float32', value=0.0) - - step_idx = layers.fill_constant( - shape=[1], dtype='int32', value=0) - cond = layers.less_than( - x=step_idx, y=max_length) # default force_cpu=True - - parent_idx = layers.fill_constant([1], dtype='int32', value=0) - while_op = layers.While(cond) - - def compute_topk_scores_and_seq(sequences, - scores, - scores_to_gather, - flags, - beam_size, - select_beam=None, - generate_id=None): - scores = layers.reshape(scores, shape=[1, -1]) - _, topk_indexs = layers.topk(scores, k=beam_size) - - topk_indexs = layers.reshape(topk_indexs, shape=[-1]) - - # gather result - - top_seq = layers.gather(sequences, topk_indexs) - topk_flags = layers.gather(flags, topk_indexs) - topk_gather_scores = layers.gather(scores_to_gather, - topk_indexs) - - if select_beam: - topk_beam = layers.gather(select_beam, topk_indexs) - else: - topk_beam = select_beam - - if generate_id: - topk_id = layers.gather(generate_id, topk_indexs) - else: - topk_id = generate_id - return top_seq, topk_gather_scores, topk_flags, topk_beam, topk_id - - def grow_alive(curr_seq, curr_scores, curr_log_probs, - curr_finished, select_beam, generate_id): - curr_scores += curr_finished * -INF - return compute_topk_scores_and_seq( - curr_seq, - curr_scores, - curr_log_probs, - curr_finished, - beam_size, - select_beam, - generate_id=generate_id) - - def grow_finished(finished_seq, finished_scores, finished_flag, - curr_seq, curr_scores, curr_finished): - finished_seq = layers.concat( - [ - finished_seq, layers.fill_constant( - [beam_size, 1], dtype='int64', value=1) - ], - axis=1) - curr_scores += (1.0 - curr_finished) * -INF - #layers.Print( curr_scores, message="curr scores") - curr_finished_seq = layers.concat( - [finished_seq, curr_seq], axis=0) - curr_finished_scores = layers.concat( - [finished_scores, curr_scores], axis=0) - curr_finished_flags = layers.concat( - [finished_flag, curr_finished], axis=0) - - return compute_topk_scores_and_seq( - curr_finished_seq, curr_finished_scores, - curr_finished_scores, curr_finished_flags, beam_size) - - def is_finished(alive_log_prob, finished_scores, - finished_in_finished): - - max_out_len = 200 - max_length_penalty = layers.pow(layers.fill_constant( - [1], dtype='float32', value=( - (5.0 + max_out_len) / 6.0)), - alpha) - - lower_bound_alive_score = layers.slice( - alive_log_prob, starts=[0], ends=[1], - axes=[0]) / max_length_penalty - - lowest_score_of_fininshed_in_finished = finished_scores * finished_in_finished - lowest_score_of_fininshed_in_finished += ( - 1.0 - finished_in_finished) * -INF - lowest_score_of_fininshed_in_finished = layers.reduce_min( - lowest_score_of_fininshed_in_finished) - - met = layers.less_than( - lower_bound_alive_score, - lowest_score_of_fininshed_in_finished) - met = layers.cast(met, 'float32') - bound_is_met = layers.reduce_sum(met) - - finished_eos_num = layers.reduce_sum(finished_in_finished) - - finish_cond = layers.less_than( - finished_eos_num, - layers.fill_constant( - [1], dtype='float32', value=beam_size)) - - return finish_cond - - def grow_top_k(step_idx, alive_seq, alive_log_prob, parant_idx): - pre_ids = alive_seq - - dec_step_emb = layers.embedding( - input=pre_ids, - size=[self.tar_vocab_size, self.hidden_size], - dtype='float32', - is_sparse=False, - param_attr=fluid.ParamAttr( - name='target_embedding', - initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale))) - - dec_att_out, new_hidden_array, new_cell_array = decoder_step( - dec_step_emb, pre_hidden_array, pre_cell_array) - - projection = layers.matmul(dec_att_out, softmax_weight) - - logits = layers.softmax(projection) - current_log = layers.elementwise_add( - x=layers.log(logits), y=alive_log_prob, axis=0) - base_1 = layers.cast(step_idx, 'float32') + 6.0 - base_1 /= 6.0 - length_penalty = layers.pow(base_1, alpha) - - len_pen = layers.pow((( - 5. + layers.cast(step_idx + 1, 'float32')) / 6.), alpha) - - current_log = layers.reshape(current_log, shape=[1, -1]) - - current_log = current_log / length_penalty - topk_scores, topk_indices = layers.topk( - input=current_log, k=beam_size) - - topk_scores = layers.reshape(topk_scores, shape=[-1]) - - topk_log_probs = topk_scores * length_penalty - - generate_id = layers.reshape( - topk_indices, shape=[-1]) % self.tar_vocab_size - - selected_beam = layers.reshape( - topk_indices, shape=[-1]) // self.tar_vocab_size - - topk_finished = layers.equal(generate_id, eos_ids) - - topk_finished = layers.cast(topk_finished, 'float32') - - generate_id = layers.reshape(generate_id, shape=[-1, 1]) - - pre_tokens_list = layers.gather(tokens, selected_beam) - - full_tokens_list = layers.concat( - [pre_tokens_list, generate_id], axis=1) - - - return full_tokens_list, topk_log_probs, topk_scores, topk_finished, selected_beam, generate_id, \ - dec_att_out, new_hidden_array, new_cell_array - - with while_op.block(): - topk_seq, topk_log_probs, topk_scores, topk_finished, topk_beam, topk_generate_id, attention_out, new_hidden_array, new_cell_array = \ - grow_top_k( step_idx, pre_tokens, pre_score, parent_idx) - alive_seq, alive_log_prob, _, alive_beam, alive_id = grow_alive( - topk_seq, topk_scores, topk_log_probs, topk_finished, - topk_beam, topk_generate_id) - - finished_seq_2, finished_scores_2, finished_flags_2, _, _ = grow_finished( - finished_seq, finished_scores, finished_flag, topk_seq, - topk_scores, topk_finished) - - finished_cond = is_finished( - alive_log_prob, finished_scores_2, finished_flags_2) - - layers.increment(x=step_idx, value=1.0, in_place=True) - - layers.assign(alive_beam, parent_idx) - layers.assign(alive_id, pre_tokens) - layers.assign(alive_log_prob, pre_score) - layers.assign(alive_seq, tokens) - layers.assign(finished_seq_2, finished_seq) - layers.assign(finished_scores_2, finished_scores) - layers.assign(finished_flags_2, finished_flag) - - # update init_hidden, init_cell, input_feed - new_feed = layers.gather(attention_out, parent_idx) - layers.assign(new_feed, pre_feed) - for i in range(self.num_layers): - new_hidden_var = layers.gather(new_hidden_array[i], - parent_idx) - layers.assign(new_hidden_var, pre_hidden_array[i]) - new_cell_var = layers.gather(new_cell_array[i], - parent_idx) - layers.assign(new_cell_var, pre_cell_array[i]) - - length_cond = layers.less_than(x=step_idx, y=max_length) - layers.logical_and(x=length_cond, y=finished_cond, out=cond) - - tokens_with_eos = tokens - - all_seq = layers.concat([tokens_with_eos, finished_seq], axis=0) - all_score = layers.concat([pre_score, finished_scores], axis=0) - _, topk_index = layers.topk(all_score, k=beam_size) - topk_index = layers.reshape(topk_index, shape=[-1]) - final_seq = layers.gather(all_seq, topk_index) - final_score = layers.gather(all_score, topk_index) - - return final_seq - elif mode == 'greedy_search': - max_src_seq_len = layers.shape(self.src)[1] - max_length = max_src_seq_len * 2 - #max_length = layers.fill_constant( [1], dtype='int32', value = 10) - pre_ids = layers.fill_constant([1, 1], dtype='int64', value=1) - full_ids = layers.fill_constant([1, 1], dtype='int64', value=1) - - score = layers.fill_constant([1], dtype='float32', value=0.0) - - eos_ids = layers.fill_constant([1, 1], dtype='int64', value=2) - - pre_hidden_array = [] - pre_cell_array = [] - pre_feed = layers.fill_constant( - [1, self.hidden_size], dtype='float32', value=0) - for i in range(self.num_layers): - pre_hidden_array.append(enc_last_hidden[i]) - pre_cell_array.append(enc_last_cell[i]) - #pre_hidden_array.append( layers.fill_constant( [1, hidden_size], dtype='float32', value=0) ) - #pre_cell_array.append( layers.fill_constant( [1, hidden_size], dtype='float32', value=0) ) - - step_idx = layers.fill_constant( - shape=[1], dtype='int32', value=0) - cond = layers.less_than( - x=step_idx, y=max_length) # default force_cpu=True - while_op = layers.While(cond) - - with while_op.block(): - - dec_step_emb = layers.embedding( - input=pre_ids, - size=[self.tar_vocab_size, self.hidden_size], - dtype='float32', - is_sparse=False, - param_attr=fluid.ParamAttr( - name='target_embedding', - initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale))) - - dec_att_out, new_hidden_array, new_cell_array = decoder_step( - dec_step_emb, pre_hidden_array, pre_cell_array) - - projection = layers.matmul(dec_att_out, softmax_weight) - - logits = layers.softmax(projection) - logits = layers.log(logits) - - current_log = layers.elementwise_add(logits, score, axis=0) - - topk_score, topk_indices = layers.topk( - input=current_log, k=1) - - new_ids = layers.concat([full_ids, topk_indices]) - layers.assign(new_ids, full_ids) - #layers.Print( full_ids, message="ful ids") - layers.assign(topk_score, score) - layers.assign(topk_indices, pre_ids) - layers.assign(dec_att_out, pre_feed) - for i in range(self.num_layers): - layers.assign(new_hidden_array[i], pre_hidden_array[i]) - layers.assign(new_cell_array[i], pre_cell_array[i]) - - layers.increment(x=step_idx, value=1.0, in_place=True) - - eos_met = layers.not_equal(topk_indices, eos_ids) - length_cond = layers.less_than(x=step_idx, y=max_length) - layers.logical_and(x=length_cond, y=eos_met, out=cond) - - return full_ids - - raise Exception("error") - else: - print("mode not supprt", mode) - - def _compute_loss(self, dec_output): - loss = layers.softmax_with_cross_entropy( - logits=dec_output, label=self.label, soft_label=False) - - loss = layers.reshape(loss, shape=[self.batch_size, -1]) - - max_tar_seq_len = layers.shape(self.tar)[1] - tar_mask = layers.sequence_mask( - self.tar_sequence_length, maxlen=max_tar_seq_len, dtype='float32') - loss = loss * tar_mask - loss = layers.reduce_mean(loss, dim=[0]) - loss = layers.reduce_sum(loss) - - loss.permissions = True - - return loss - - def _beam_search(self, enc_last_hidden, enc_last_cell): - pass - - def build_graph(self, mode='train', beam_size=10): - if mode == 'train' or mode == 'eval': - self._build_data() - self._emebdding() - enc_output, enc_last_hidden, enc_last_cell = self._build_encoder() - dec_output = self._build_decoder(enc_last_hidden, enc_last_cell) - - loss = self._compute_loss(dec_output) - return loss - elif mode == "beam_search" or mode == 'greedy_search': - self._build_data() - self._emebdding() - enc_output, enc_last_hidden, enc_last_cell = self._build_encoder() - dec_output = self._build_decoder( - enc_last_hidden, enc_last_cell, mode=mode, beam_size=beam_size) - - return dec_output - else: - print("not support mode ", mode) - raise Exception("not support mode: " + mode) diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/data/download_en-vi.sh b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/data/download_en-vi.sh deleted file mode 100755 index ae61044bcd34b84c35cf252871535be2fecb7a2e..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/data/download_en-vi.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# IWSLT15 Vietnames to English is a small dataset contain 133k parallel data -# this script download the data from stanford website -# -# Usage: -# ./download_en-vi.sh output_path -# -# If output_path is not specified, a dir nameed "./en_vi" will be created and used as -# output path - -set -ex -OUTPUT_PATH="${1:-en-vi}" -SITE_PATH="https://nlp.stanford.edu/projects/nmt/data" - -mkdir -v -p $OUTPUT_PATH - -# Download iwslt15 small dataset from standford website. -echo "Begin to download training dataset train.en and train.vi." -wget "$SITE_PATH/iwslt15.en-vi/train.en" -O "$OUTPUT_PATH/train.en" -wget "$SITE_PATH/iwslt15.en-vi/train.vi" -O "$OUTPUT_PATH/train.vi" - -echo "Begin to download dev dataset tst2012.en and tst2012.vi." -wget "$SITE_PATH/iwslt15.en-vi/tst2012.en" -O "$OUTPUT_PATH/tst2012.en" -wget "$SITE_PATH/iwslt15.en-vi/tst2012.vi" -O "$OUTPUT_PATH/tst2012.vi" - -echo "Begin to download test dataset tst2013.en and tst2013.vi." -wget "$SITE_PATH/iwslt15.en-vi/tst2013.en" -O "$OUTPUT_PATH/tst2013.en" -wget "$SITE_PATH/iwslt15.en-vi/tst2013.vi" -O "$OUTPUT_PATH/tst2013.vi" - -echo "Begin to ownload vocab file vocab.en and vocab.vi." -wget "$SITE_PATH/iwslt15.en-vi/vocab.en" -O "$OUTPUT_PATH/vocab.en" -wget "$SITE_PATH/iwslt15.en-vi/vocab.vi" -O "$OUTPUT_PATH/vocab.vi" - diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/infer.py b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/infer.py deleted file mode 100644 index 9ac4c73e0a289a99267e2c6166b1bb06ff8430db..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/infer.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import time -import os -import random - -import math - -import paddle -import paddle.fluid as fluid -import paddle.fluid.framework as framework -from paddle.fluid.executor import Executor - -import reader - -import sys -if sys.version[0] == '2': - reload(sys) - sys.setdefaultencoding("utf-8") -import os - -from args import * -#from . import lm_model -import logging -import pickle - -from attention_model import AttentionModel - -from base_model import BaseModel - -SEED = 123 - - -def train(): - args = parse_args() - - num_layers = args.num_layers - src_vocab_size = args.src_vocab_size - tar_vocab_size = args.tar_vocab_size - batch_size = args.batch_size - dropout = args.dropout - init_scale = args.init_scale - max_grad_norm = args.max_grad_norm - hidden_size = args.hidden_size - # inference process - - print("src", src_vocab_size) - - # dropout type using upscale_in_train, dropout can be remove in inferecen - # So we can set dropout to 0 - if args.attention: - model = AttentionModel( - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=num_layers, - init_scale=init_scale, - dropout=0.0) - else: - model = BaseModel( - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=num_layers, - init_scale=init_scale, - dropout=0.0) - - beam_size = args.beam_size - trans_res = model.build_graph(mode='beam_search', beam_size=beam_size) - # clone from default main program and use it as the validation program - main_program = fluid.default_main_program() - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = Executor(place) - exe.run(framework.default_startup_program()) - - source_vocab_file = args.vocab_prefix + "." + args.src_lang - infer_file = args.infer_file - - infer_data = reader.raw_mono_data(source_vocab_file, infer_file) - - def prepare_input(batch, epoch_id=0, with_lr=True): - src_ids, src_mask, tar_ids, tar_mask = batch - res = {} - src_ids = src_ids.reshape((src_ids.shape[0], src_ids.shape[1], 1)) - in_tar = tar_ids[:, :-1] - label_tar = tar_ids[:, 1:] - - in_tar = in_tar.reshape((in_tar.shape[0], in_tar.shape[1], 1)) - in_tar = np.zeros_like(in_tar, dtype='int64') - label_tar = label_tar.reshape( - (label_tar.shape[0], label_tar.shape[1], 1)) - label_tar = np.zeros_like(label_tar, dtype='int64') - - res['src'] = src_ids - res['tar'] = in_tar - res['label'] = label_tar - res['src_sequence_length'] = src_mask - res['tar_sequence_length'] = tar_mask - - return res, np.sum(tar_mask) - - dir_name = args.reload_model - print("dir name", dir_name) - fluid.io.load_params(exe, dir_name) - - train_data_iter = reader.get_data_iter(infer_data, 1, mode='eval') - - tar_id2vocab = [] - tar_vocab_file = args.vocab_prefix + "." + args.tar_lang - with open(tar_vocab_file, "r") as f: - for line in f.readlines(): - tar_id2vocab.append(line.strip()) - - infer_output_file = args.infer_output_file - - out_file = open(infer_output_file, 'w') - - for batch_id, batch in enumerate(train_data_iter): - input_data_feed, word_num = prepare_input(batch, epoch_id=0) - - fetch_outs = exe.run(feed=input_data_feed, - fetch_list=[trans_res.name], - use_program_cache=False) - - res = [tar_id2vocab[e] for e in fetch_outs[0].reshape(-1)] - - res = res[1:] - - new_res = [] - for ele in res: - if ele == "": - break - new_res.append(ele) - - out_file.write(' '.join(new_res)) - out_file.write('\n') - - out_file.close() - - -if __name__ == '__main__': - train() diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/infer.sh b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/infer.sh deleted file mode 100644 index ffd48da6adc4ecc080f504d3b6a6a244fa6eedd9..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/infer.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -ex -export CUDA_VISIBLE_DEVICES=0 - -python infer.py \ - --src_lang en --tar_lang vi \ - --attention True \ - --num_layers 2 \ - --hidden_size 512 \ - --src_vocab_size 17191 \ - --tar_vocab_size 7709 \ - --batch_size 128 \ - --dropout 0.2 \ - --init_scale 0.1 \ - --max_grad_norm 5.0 \ - --vocab_prefix data/en-vi/vocab \ - --infer_file data/en-vi/tst2013.en \ - --reload_model ./model/epoch_10 \ - --use_gpu True - diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/reader.py b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/reader.py deleted file mode 100644 index 06cc8f50b2b2adba2dd709679bbccefdcbbb977e..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/reader.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for parsing PTB text files.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import os -import sys -import numpy as np - -Py3 = sys.version_info[0] == 3 - -UNK_ID = 0 - - -def _read_words(filename): - data = [] - with open(filename, "r") as f: - if Py3: - return f.read().replace("\n", "").split() - else: - return f.read().decode("utf-8").replace("\n", "").split() - - -def read_all_line(filenam): - data = [] - with open(filename, "r") as f: - for line in f.readlines(): - data.append(line.strip()) - - -def _build_vocab(filename): - - vocab_dict = {} - ids = 0 - with open(filename, "r") as f: - for line in f.readlines(): - vocab_dict[line.strip()] = ids - ids += 1 - - print("vocab word num", ids) - - return vocab_dict - - -def _para_file_to_ids(src_file, tar_file, src_vocab, tar_vocab): - - src_data = [] - with open(src_file, "r") as f_src: - for line in f_src.readlines(): - arra = line.strip().split() - ids = [src_vocab[w] if w in src_vocab else UNK_ID for w in arra] - ids = ids - - src_data.append(ids) - - tar_data = [] - with open(tar_file, "r") as f_tar: - for line in f_tar.readlines(): - arra = line.strip().split() - ids = [tar_vocab[w] if w in tar_vocab else UNK_ID for w in arra] - - ids = [1] + ids + [2] - - tar_data.append(ids) - - return src_data, tar_data - - -def filter_len(src, tar, max_sequence_len=50): - new_src = [] - new_tar = [] - - for id1, id2 in zip(src, tar): - if len(id1) > max_sequence_len: - id1 = id1[:max_sequence_len] - if len(id2) > max_sequence_len + 2: - id2 = id2[:max_sequence_len + 2] - - new_src.append(id1) - new_tar.append(id2) - - return new_src, new_tar - - -def raw_data(src_lang, - tar_lang, - vocab_prefix, - train_prefix, - eval_prefix, - test_prefix, - max_sequence_len=50): - - src_vocab_file = vocab_prefix + "." + src_lang - tar_vocab_file = vocab_prefix + "." + tar_lang - - src_train_file = train_prefix + "." + src_lang - tar_train_file = train_prefix + "." + tar_lang - - src_eval_file = eval_prefix + "." + src_lang - tar_eval_file = eval_prefix + "." + tar_lang - - src_test_file = test_prefix + "." + src_lang - tar_test_file = test_prefix + "." + tar_lang - - src_vocab = _build_vocab(src_vocab_file) - tar_vocab = _build_vocab(tar_vocab_file) - - train_src, train_tar = _para_file_to_ids( src_train_file, tar_train_file, \ - src_vocab, tar_vocab ) - train_src, train_tar = filter_len( - train_src, train_tar, max_sequence_len=max_sequence_len) - eval_src, eval_tar = _para_file_to_ids( src_eval_file, tar_eval_file, \ - src_vocab, tar_vocab ) - - test_src, test_tar = _para_file_to_ids( src_test_file, tar_test_file, \ - src_vocab, tar_vocab ) - - return ( train_src, train_tar), (eval_src, eval_tar), (test_src, test_tar),\ - (src_vocab, tar_vocab) - - -def raw_mono_data(vocab_file, file_path): - - src_vocab = _build_vocab(vocab_file) - - test_src, test_tar = _para_file_to_ids( file_path, file_path, \ - src_vocab, src_vocab ) - - return (test_src, test_tar) - - -def get_data_iter(raw_data, batch_size, mode='train', enable_ce=False): - - src_data, tar_data = raw_data - - data_len = len(src_data) - - index = np.arange(data_len) - if mode == "train" and not enable_ce: - np.random.shuffle(index) - - def to_pad_np(data, source=False): - max_len = 0 - for ele in data: - if len(ele) > max_len: - max_len = len(ele) - - ids = np.ones((batch_size, max_len), dtype='int64') * 2 - mask = np.zeros((batch_size), dtype='int32') - - for i, ele in enumerate(data): - ids[i, :len(ele)] = ele - if not source: - mask[i] = len(ele) - 1 - else: - mask[i] = len(ele) - - return ids, mask - - b_src = [] - - cache_num = 20 - if mode != "train": - cache_num = 1 - for j in range(data_len): - if len(b_src) == batch_size * cache_num: - # build batch size - - # sort - new_cache = sorted(b_src, key=lambda k: len(k[0])) - - for i in range(cache_num): - batch_data = new_cache[i * batch_size:(i + 1) * batch_size] - src_cache = [w[0] for w in batch_data] - tar_cache = [w[1] for w in batch_data] - src_ids, src_mask = to_pad_np(src_cache, source=True) - tar_ids, tar_mask = to_pad_np(tar_cache) - - #print( "src ids", src_ids ) - yield (src_ids, src_mask, tar_ids, tar_mask) - - b_src = [] - - b_src.append((src_data[index[j]], tar_data[index[j]])) - if len(b_src) == batch_size * cache_num: - new_cache = sorted(b_src, key=lambda k: len(k[0])) - - for i in range(cache_num): - batch_data = new_cache[i * batch_size:(i + 1) * batch_size] - src_cache = [w[0] for w in batch_data] - tar_cache = [w[1] for w in batch_data] - src_ids, src_mask = to_pad_np(src_cache, source=True) - tar_ids, tar_mask = to_pad_np(tar_cache) - - #print( "src ids", src_ids ) - yield (src_ids, src_mask, tar_ids, tar_mask) diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/run.sh b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/run.sh deleted file mode 100644 index cf48282deea544f5ca2d233a4af7336fb664cc65..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/run.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -ex -export CUDA_VISIBLE_DEVICES=0 - -python train.py \ - --src_lang en --tar_lang vi \ - --attention True \ - --num_layers 2 \ - --hidden_size 512 \ - --src_vocab_size 17191 \ - --tar_vocab_size 7709 \ - --batch_size 128 \ - --dropout 0.2 \ - --init_scale 0.1 \ - --max_grad_norm 5.0 \ - --train_data_prefix data/en-vi/train \ - --eval_data_prefix data/en-vi/tst2012 \ - --test_data_prefix data/en-vi/tst2013 \ - --vocab_prefix data/en-vi/vocab \ - --use_gpu True - diff --git a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/train.py b/PaddleNLP/unarchived/neural_machine_translation/rnn_search/train.py deleted file mode 100644 index 820c383a0682b61c728657cdc1349fcd46d5a650..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/rnn_search/train.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import time -import os -import random -import math -import contextlib - -import paddle -import paddle.fluid as fluid -import paddle.fluid.framework as framework -import paddle.fluid.profiler as profiler -from paddle.fluid.executor import Executor - -import reader - -import sys -if sys.version[0] == '2': - reload(sys) - sys.setdefaultencoding("utf-8") -import os - -from args import * -from base_model import BaseModel -from attention_model import AttentionModel -import logging -import pickle - -SEED = 123 - - -@contextlib.contextmanager -def profile_context(profile=True): - if profile: - with profiler.profiler('All', 'total', 'seq2seq.profile'): - yield - else: - yield - - -def main(): - args = parse_args() - - num_layers = args.num_layers - src_vocab_size = args.src_vocab_size - tar_vocab_size = args.tar_vocab_size - batch_size = args.batch_size - dropout = args.dropout - init_scale = args.init_scale - max_grad_norm = args.max_grad_norm - hidden_size = args.hidden_size - - if args.enable_ce: - fluid.default_main_program().random_seed = 102 - framework.default_startup_program().random_seed = 102 - - # Training process - - if args.attention: - model = AttentionModel( - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) - else: - model = BaseModel( - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) - - loss = model.build_graph() - # clone from default main program and use it as the validation program - main_program = fluid.default_main_program() - inference_program = fluid.default_main_program().clone(for_test=True) - - fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByGlobalNorm( - clip_norm=max_grad_norm)) - - lr = args.learning_rate - opt_type = args.optimizer - if opt_type == "sgd": - optimizer = fluid.optimizer.SGD(lr) - elif opt_type == "adam": - optimizer = fluid.optimizer.Adam(lr) - else: - print("only support [sgd|adam]") - raise Exception("opt type not support") - - optimizer.minimize(loss) - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = Executor(place) - exe.run(framework.default_startup_program()) - - device_count = len(fluid.cuda_places()) if args.use_gpu else len( - fluid.cpu_places()) - if device_count > 1: - raise Exception("Training using multi-GPUs is not supported now.") - - exec_strategy = fluid.ExecutionStrategy() - exec_strategy.num_threads = device_count - exec_strategy.num_iteration_per_drop_scope = 100 - - build_strategy = fluid.BuildStrategy() - build_strategy.enable_inplace = True - build_strategy.memory_optimize = False -# build_strategy.fuse_all_optimizer_ops = True - - if args.parallel: - train_program = fluid.compiler.CompiledProgram( - framework.default_main_program()).with_data_parallel( - loss_name=loss.name, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - else: - train_program = framework.default_main_program() - - train_data_prefix = args.train_data_prefix - eval_data_prefix = args.eval_data_prefix - test_data_prefix = args.test_data_prefix - vocab_prefix = args.vocab_prefix - src_lang = args.src_lang - tar_lang = args.tar_lang - print("begin to load data") - raw_data = reader.raw_data(src_lang, tar_lang, vocab_prefix, - train_data_prefix, eval_data_prefix, - test_data_prefix, args.max_len) - print("finished load data") - train_data, valid_data, test_data, _ = raw_data - - def prepare_input(batch, epoch_id=0, with_lr=True): - src_ids, src_mask, tar_ids, tar_mask = batch - res = {} - src_ids = src_ids.reshape((src_ids.shape[0], src_ids.shape[1], 1)) - in_tar = tar_ids[:, :-1] - label_tar = tar_ids[:, 1:] - - in_tar = in_tar.reshape((in_tar.shape[0], in_tar.shape[1], 1)) - label_tar = label_tar.reshape( - (label_tar.shape[0], label_tar.shape[1], 1)) - - res['src'] = src_ids - res['tar'] = in_tar - res['label'] = label_tar - res['src_sequence_length'] = src_mask - res['tar_sequence_length'] = tar_mask - - return res, np.sum(tar_mask) - - # get train epoch size - def eval(data, epoch_id=0): - eval_data_iter = reader.get_data_iter(data, batch_size, mode='eval') - total_loss = 0.0 - word_count = 0.0 - for batch_id, batch in enumerate(eval_data_iter): - input_data_feed, word_num = prepare_input( - batch, epoch_id, with_lr=False) - fetch_outs = exe.run(inference_program, - feed=input_data_feed, - fetch_list=[loss.name], - use_program_cache=False) - - cost_train = np.array(fetch_outs[0]) - - total_loss += cost_train * batch_size - word_count += word_num - - ppl = np.exp(total_loss / word_count) - - return ppl - - def train(): - ce_time = [] - ce_ppl = [] - max_epoch = args.max_epoch - for epoch_id in range(max_epoch): - start_time = time.time() - if args.enable_ce: - train_data_iter = reader.get_data_iter(train_data, batch_size, enable_ce=True) - else: - train_data_iter = reader.get_data_iter(train_data, batch_size) - - - total_loss = 0 - word_count = 0.0 - batch_times = [] - for batch_id, batch in enumerate(train_data_iter): - batch_start_time = time.time() - input_data_feed, word_num = prepare_input(batch, epoch_id=epoch_id) - fetch_outs = exe.run(program=train_program, - feed=input_data_feed, - fetch_list=[loss.name], - use_program_cache=True) - - cost_train = np.array(fetch_outs[0]) - - total_loss += cost_train * batch_size - word_count += word_num - batch_end_time = time.time() - batch_time = batch_end_time - batch_start_time - batch_times.append(batch_time) - - if batch_id > 0 and batch_id % 100 == 0: - print( - "-- Epoch:[%d]; Batch:[%d]; Time: %.5f s; ppl: %.5f" - % (epoch_id, batch_id, batch_time, np.exp(total_loss / word_count))) - ce_ppl.append(np.exp(total_loss / word_count)) - total_loss = 0.0 - word_count = 0.0 - - end_time = time.time() - epoch_time = end_time - start_time - ce_time.append(epoch_time) - print( - "\nTrain epoch:[%d]; Epoch Time: %.5f; avg_time: %.5f s/step\n" - % (epoch_id, epoch_time, sum(batch_times) / len(batch_times))) - - if not args.profile: - dir_name = args.model_path + "/epoch_" + str(epoch_id) - print("begin to save", dir_name) - fluid.io.save_params(exe, dir_name) - print("save finished") - dev_ppl = eval(valid_data) - print("dev ppl", dev_ppl) - test_ppl = eval(test_data) - print("test ppl", test_ppl) - - if args.enable_ce: - card_num = get_cards() - _ppl = 0 - _time = 0 - try: - _time = ce_time[-1] - _ppl = ce_ppl[-1] - except: - print("ce info error") - print("kpis\ttrain_duration_card%s\t%s" % - (card_num, _time)) - print("kpis\ttrain_ppl_card%s\t%f" % - (card_num, _ppl)) - - with profile_context(args.profile): - train() - - -def get_cards(): - num = 0 - cards = os.environ.get('CUDA_VISIBLE_DEVICES', '') - if cards != '': - num = len(cards.split(",")) - return num - - -if __name__ == '__main__': - main() diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/.gitignore b/PaddleNLP/unarchived/neural_machine_translation/transformer/.gitignore deleted file mode 100644 index 0d20b6487c61e7d1bde93acf4a14b7a89083a16d..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pyc diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/.run_ce.sh b/PaddleNLP/unarchived/neural_machine_translation/transformer/.run_ce.sh deleted file mode 100644 index fe721aee82781465d1cce21eadd87f57f113bd15..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/.run_ce.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -DATA_PATH=$HOME/.cache/paddle/dataset/wmt16 -if [ ! -e $DATA_PATH/en_10000.dict ] ; then - python -c 'import paddle;paddle.dataset.wmt16.train(10000, 10000, "en")().next()' - tar -zxf $DATA_PATH/wmt16.tar.gz -C $DATA_PATH -fi - -train(){ - python -u train.py \ - --src_vocab_fpath $DATA_PATH/en_10000.dict \ - --trg_vocab_fpath $DATA_PATH/de_10000.dict \ - --special_token '' '' '' \ - --train_file_pattern $DATA_PATH/wmt16/train \ - --val_file_pattern $DATA_PATH/wmt16/val \ - --use_token_batch True \ - --batch_size 2048 \ - --sort_type pool \ - --pool_size 10000 \ - --enable_ce True \ - weight_sharing False \ - pass_num 20 \ - dropout_seed 10 -} - -cudaid=${transformer:=0} # use 0-th card as default -export CUDA_VISIBLE_DEVICES=$cudaid - -train | python _ce.py - -cudaid=${transformer_m:=0,1,2,3} # use 0,1,2,3 card as default -export CUDA_VISIBLE_DEVICES=$cudaid - -train | python _ce.py diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/README.md b/PaddleNLP/unarchived/neural_machine_translation/transformer/README.md deleted file mode 100644 index 6fea167b5e7c3e9dd759ef30d9225b451350e889..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/README.md +++ /dev/null @@ -1,23 +0,0 @@ -The minimum PaddlePaddle version needed for the code sample in this directory is the lastest develop branch. If you are on a version of PaddlePaddle earlier than this, [please update your installation](http://www.paddlepaddle.org/docs/develop/documentation/en/build_and_install/pip_install_en.html). - ---- - -# Attention is All You Need: A Paddle Fluid implementation - -This is a Paddle Fluid implementation of the Transformer model in [Attention is All You Need]() (Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser, Illia Polosukhin, arxiv, 2017). - -If you use the dataset/code in your research, please cite the paper: - -```text -@inproceedings{vaswani2017attention, - title={Attention is all you need}, - author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia}, - booktitle={Advances in Neural Information Processing Systems}, - pages={6000--6010}, - year={2017} -} -``` - -### TODO - -This project is still under active development. diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/README_cn.md b/PaddleNLP/unarchived/neural_machine_translation/transformer/README_cn.md deleted file mode 100644 index bdac7cb0b7c4f9d51bbc281b351232c6edc75a36..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/README_cn.md +++ /dev/null @@ -1,272 +0,0 @@ -运行本目录下的程序示例需要使用 PaddlePaddle 最新的 develop branch 版本。如果您的 PaddlePaddle 安装版本低于此要求，请按照[安装文档](http://www.paddlepaddle.org/docs/develop/documentation/zh/build_and_install/pip_install_cn.html)中的说明更新 PaddlePaddle 安装版本。 - ---- - -## Transformer - -以下是本例的简要目录结构及说明： - -```text -. -├── images # README 文档中的图片 -├── config.py # 训练、预测以及模型参数配置 -├── infer.py # 预测脚本 -├── model.py # 模型定义 -├── optim.py # learning rate scheduling 计算程序 -├── reader.py # 数据读取接口 -├── README.md # 文档 -├── train.py # 训练脚本 -└── gen_data.sh # 数据生成脚本 -``` - -### 简介 - -Transformer 是论文 [Attention Is All You Need](https://arxiv.org/abs/1706.03762) 中提出的用以完成机器翻译（machine translation, MT）等序列到序列（sequence to sequence, Seq2Seq）学习任务的一种全新网络结构，其完全使用注意力（Attention）机制来实现序列到序列的建模[1]。 - -相较于此前 Seq2Seq 模型中广泛使用的循环神经网络（Recurrent Neural Network, RNN），使用（Self）Attention 进行输入序列到输出序列的变换主要具有以下优势： - -- 计算复杂度小 - - 特征维度为 d 、长度为 n 的序列，在 RNN 中计算复杂度为 `O(n * d * d)` （n 个时间步，每个时间步计算 d 维的矩阵向量乘法），在 Self-Attention 中计算复杂度为 `O(n * n * d)` （n 个时间步两两计算 d 维的向量点积或其他相关度函数），n 通常要小于 d 。 -- 计算并行度高 - - RNN 中当前时间步的计算要依赖前一个时间步的计算结果；Self-Attention 中各时间步的计算只依赖输入不依赖之前时间步输出，各时间步可以完全并行。 -- 容易学习长程依赖（long-range dependencies） - - RNN 中相距为 n 的两个位置间的关联需要 n 步才能建立；Self-Attention 中任何两个位置都直接相连；路径越短信号传播越容易。 - -这些也在机器翻译任务中得到了印证，Transformer 模型在训练时间大幅减少的同时取得了 WMT'14 英德翻译任务 BLEU 值的新高。此外，Transformer 在应用于成分句法分析（Constituency Parsing）任务时也有着不俗的表现，这也说明其具有较高的通用性，容易迁移到其他应用场景中。这些都表明 Transformer 有着广阔的前景。 - -### 模型概览 - -Transformer 同样使用了 Seq2Seq 模型中典型的编码器-解码器（Encoder-Decoder）的框架结构，整体网络结构如图1所示。 - -

-
-图 1. Transformer 网络结构图 -

- -Encoder 由若干相同的 layer 堆叠组成，每个 layer 主要由多头注意力（Multi-Head Attention）和全连接的前馈（Feed-Forward）网络这两个 sub-layer 构成。 -- Multi-Head Attention 在这里用于实现 Self-Attention，相比于简单的 Attention 机制，其将输入进行多路线性变换后分别计算 Attention 的结果，并将所有结果拼接后再次进行线性变换作为输出。参见图2，其中 Attention 使用的是点积（Dot-Product），并在点积后进行了 scale 的处理以避免因点积结果过大进入 softmax 的饱和区域。 -- Feed-Forward 网络会对序列中的每个位置进行相同的计算（Position-wise），其采用的是两次线性变换中间加以 ReLU 激活的结构。 - -此外，每个 sub-layer 后还施以 Residual Connection [2]和 Layer Normalization [3]来促进梯度传播和模型收敛。 - -

-
-图 2. Multi-Head Attention -

- -Decoder 具有和 Encoder 类似的结构，只是相比于组成 Encoder 的 layer ，在组成 Decoder 的 layer 中还多了一个 Multi-Head Attention 的 sub-layer 来实现对 Encoder 输出的 Attention，这个 Encoder-Decoder Attention 在其他 Seq2Seq 模型中也是存在的。 - - -### 数据准备 - -WMT 数据集是机器翻译领域公认的主流数据集，[WMT'16 EN-DE 数据集](http://www.statmt.org/wmt16/translation-task.html)是其中一个中等规模的数据集，也是 Transformer 论文中用到的一个数据集，这里将其作为示例，可以直接运行 `gen_data.sh` 脚本进行 WMT'16 EN-DE 数据集的下载和预处理。数据处理过程主要包括 Tokenize 和 BPE 编码（byte-pair encoding）；BPE 编码的数据能够较好的解决未登录词（out-of-vocabulary，OOV）的问题[4]，其在 Transformer 论文中也被使用。运行成功后，将会生成文件夹 `gen_data`，其目录结构如下（可在 `gen_data.sh` 中修改）： - -```text -. -├── wmt16_ende_data # WMT16 英德翻译数据 -├── wmt16_ende_data_bpe # BPE 编码的 WMT16 英德翻译数据 -├── mosesdecoder # Moses 机器翻译工具集，包含了 Tokenize、BLEU 评估等脚本 -└── subword-nmt # BPE 编码的代码 -``` - -`gen_data/wmt16_ende_data_bpe` 中是我们最终使用的英德翻译数据，其中 `train.tok.clean.bpe.32000.en-de` 为训练数据，`newstest2016.tok.bpe.32000.en-de` 等为验证和测试数据，`vocab_all.bpe.32000` 为相应的词典文件（已加入 `` 、`` 和 `` 这三个特殊符号，源语言和目标语言共享该词典文件）。另外我们也整理提供了一份处理好的 WMT'16 EN-DE 数据以供[下载](https://transformer-res.bj.bcebos.com/wmt16_ende_data_bpe_clean.tar.gz)使用（包含训练所需 BPE 数据和词典以及预测和评估所需的 BPE 数据和 tokenize 的数据）。 - -对于其他自定义数据，转换为类似 `train.tok.clean.bpe.32000.en-de` 的数据格式（`\t` 分隔的源语言和目标语言句子对，句子中的 token 之间使用空格分隔）即可；如需使用 BPE 编码，亦可以使用类似 WMT'16 EN-DE 原始数据的格式，参照 `gen_data.sh` 进行处理。 - -### 模型训练 - -`train.py` 是模型训练脚本。以英德翻译数据为例，可以执行以下命令进行模型训练： -```sh -python -u train.py \ - --src_vocab_fpath gen_data/wmt16_ende_data_bpe/vocab_all.bpe.32000 \ - --trg_vocab_fpath gen_data/wmt16_ende_data_bpe/vocab_all.bpe.32000 \ - --special_token '' '' '' \ - --train_file_pattern gen_data/wmt16_ende_data_bpe/train.tok.clean.bpe.32000.en-de \ - --token_delimiter ' ' \ - --use_token_batch True \ - --batch_size 4096 \ - --sort_type pool \ - --pool_size 200000 -``` -上述命令中设置了源语言词典文件路径（`src_vocab_fpath`）、目标语言词典文件路径（`trg_vocab_fpath`）、训练数据文件（`train_file_pattern`，支持通配符）等数据相关的参数和构造 batch 方式（`use_token_batch` 指定了数据按照 token 数目或者 sequence 数目组成 batch）等 reader 相关的参数。有关这些参数更详细的信息可以通过执行以下命令查看： -```sh -python train.py --help -``` - -更多模型训练相关的参数则在 `config.py` 中的 `ModelHyperParams` 和 `TrainTaskConfig` 内定义；`ModelHyperParams` 定义了 embedding 维度等模型超参数，`TrainTaskConfig` 定义了 warmup 步数等训练需要的参数。这些参数默认使用了 Transformer 论文中 base model 的配置，如需调整可以在该脚本中进行修改。另外这些参数同样可在执行训练脚本的命令行中设置，传入的配置会合并并覆盖 `config.py` 中的配置，如可以通过以下命令来训练 Transformer 论文中的 big model （如显存不够可适当减小 batch size 的值，或设置 `max_length 200` 过滤过长的句子，或修改某些显存使用相关环境变量的值）： - -```sh -# 显存使用的比例，显存不足可适当增大，最大为1 -export FLAGS_fraction_of_gpu_memory_to_use=1.0 -# 显存清理的阈值，显存不足可适当减小，最小为0，为负数时不启用 -export FLAGS_eager_delete_tensor_gb=0.8 -python -u train.py \ - --src_vocab_fpath gen_data/wmt16_ende_data_bpe/vocab_all.bpe.32000 \ - --trg_vocab_fpath gen_data/wmt16_ende_data_bpe/vocab_all.bpe.32000 \ - --special_token '' '' '' \ - --train_file_pattern gen_data/wmt16_ende_data_bpe/train.tok.clean.bpe.32000.en-de \ - --token_delimiter ' ' \ - --use_token_batch True \ - --batch_size 3200 \ - --sort_type pool \ - --pool_size 200000 \ - n_head 16 \ - d_model 1024 \ - d_inner_hid 4096 \ - prepostprocess_dropout 0.3 -``` -有关这些参数更详细信息的请参考 `config.py` 中的注释说明。 - -训练时默认使用所有 GPU，可以通过 `CUDA_VISIBLE_DEVICES` 环境变量来设置使用的 GPU 数目。也可以只使用 CPU 训练(通过参数 `--divice CPU` 设置)，训练速度相对较慢。在训练过程中，每隔一定 iteration 后(通过参数 `save_freq` 设置，默认为10000)保存模型到参数 `model_dir` 指定的目录，每个 epoch 结束后也会保存 checkpiont 到 `ckpt_dir` 指定的目录，每隔一定数目的 iteration (通过参数 `--fetch_steps` 设置，默认为100)将打印如下的日志到标准输出： -```txt -[2018-10-26 00:49:24,705 INFO train.py:536] step_idx: 0, epoch: 0, batch: 0, avg loss: 10.999878, normalized loss: 9.624138, ppl: 59866.832031 -[2018-10-26 00:50:08,717 INFO train.py:545] step_idx: 100, epoch: 0, batch: 100, avg loss: 9.454134, normalized loss: 8.078394, ppl: 12760.809570, speed: 2.27 step/s -[2018-10-26 00:50:52,655 INFO train.py:545] step_idx: 200, epoch: 0, batch: 200, avg loss: 8.643907, normalized loss: 7.268166, ppl: 5675.458496, speed: 2.28 step/s -[2018-10-26 00:51:36,529 INFO train.py:545] step_idx: 300, epoch: 0, batch: 300, avg loss: 7.916654, normalized loss: 6.540914, ppl: 2742.579346, speed: 2.28 step/s -[2018-10-26 00:52:20,692 INFO train.py:545] step_idx: 400, epoch: 0, batch: 400, avg loss: 7.902879, normalized loss: 6.527138, ppl: 2705.058350, speed: 2.26 step/s -[2018-10-26 00:53:04,537 INFO train.py:545] step_idx: 500, epoch: 0, batch: 500, avg loss: 7.818271, normalized loss: 6.442531, ppl: 2485.604492, speed: 2.28 step/s -[2018-10-26 00:53:48,580 INFO train.py:545] step_idx: 600, epoch: 0, batch: 600, avg loss: 7.554341, normalized loss: 6.178601, ppl: 1909.012451, speed: 2.27 step/s -[2018-10-26 00:54:32,878 INFO train.py:545] step_idx: 700, epoch: 0, batch: 700, avg loss: 7.177765, normalized loss: 5.802025, ppl: 1309.977661, speed: 2.26 step/s -[2018-10-26 00:55:17,108 INFO train.py:545] step_idx: 800, epoch: 0, batch: 800, avg loss: 7.005494, normalized loss: 5.629754, ppl: 1102.674805, speed: 2.26 step/s -``` - -### 模型预测 - -`infer.py` 是模型预测脚本。以英德翻译数据为例，模型训练完成后可以执行以下命令对指定文件中的文本进行翻译： -```sh -python -u infer.py \ - --src_vocab_fpath gen_data/wmt16_ende_data_bpe/vocab_all.bpe.32000 \ - --trg_vocab_fpath gen_data/wmt16_ende_data_bpe/vocab_all.bpe.32000 \ - --special_token '' '' '' \ - --test_file_pattern gen_data/wmt16_ende_data_bpe/newstest2016.tok.bpe.32000.en-de \ - --token_delimiter ' ' \ - --batch_size 32 \ - model_path trained_models/iter_100000.infer.model \ - beam_size 5 \ - max_out_len 255 -``` -和模型训练时类似，预测时也需要设置数据和 reader 相关的参数，并可以执行 `python infer.py --help` 查看这些参数的说明（部分参数意义和训练时略有不同）；同样可以在预测命令中设置模型超参数，但应与模型训练时的设置一致，如训练时使用 big model 的参数设置，则预测时对应类似如下命令： -```sh -python -u infer.py \ - --src_vocab_fpath gen_data/wmt16_ende_data_bpe/vocab_all.bpe.32000 \ - --trg_vocab_fpath gen_data/wmt16_ende_data_bpe/vocab_all.bpe.32000 \ - --special_token '' '' '' \ - --test_file_pattern gen_data/wmt16_ende_data_bpe/newstest2016.tok.bpe.32000.en-de \ - --token_delimiter ' ' \ - --batch_size 32 \ - model_path trained_models/iter_100000.infer.model \ - n_head 16 \ - d_model 1024 \ - d_inner_hid 4096 \ - prepostprocess_dropout 0.3 \ - beam_size 5 \ - max_out_len 255 -``` -此外相比于模型训练，预测时还有一些额外的参数，如需要设置 `model_path` 来给出模型所在目录，可以设置 `beam_size` 和 `max_out_len` 来指定 Beam Search 算法的搜索宽度和最大深度（翻译长度），这些参数也可以在 `config.py` 中的 `InferTaskConfig` 内查阅注释说明并进行更改设置。 - -执行以上预测命令会打印翻译结果到标准输出，每行输出是对应行输入的得分最高的翻译。对于使用 BPE 的英德数据，预测出的翻译结果也将是 BPE 表示的数据，要还原成原始的数据（这里指 tokenize 后的数据）才能进行正确的评估，可以使用以下命令来恢复 `predict.txt` 内的翻译结果到 `predict.tok.txt` 中（无需再次 tokenize 处理）： -```sh -sed -r 's/(@@ )|(@@ ?$)//g' predict.txt > predict.tok.txt -``` - -接下来就可以使用参考翻译对翻译结果进行 BLEU 指标的评估了，评估需要用到 mosesdecoder 中的脚本，可以通过以下命令获取： -```sh -git clone https://github.com/moses-smt/mosesdecoder.git -``` -以英德翻译 `newstest2014.tok.de` 数据为例，获取 mosesdecoder 后使用 `multi-bleu.perl` 执行如下命令进行翻译结果评估： -```sh -perl gen_data/mosesdecoder/scripts/generic/multi-bleu.perl gen_data/wmt16_ende_data/newstest2014.tok.de < predict.tok.txt -``` -可以看到类似如下的结果： -``` -BLEU = 26.35, 57.7/32.1/20.0/13.0 (BP=1.000, ratio=1.013, hyp_len=63903, ref_len=63078) -``` -目前在未使用 model average 的情况下，英德翻译 base model 和 big model 八卡训练 100K 个 iteration 后测试 BLEU 值如下： - -| 测试集 | newstest2014 | newstest2015 | newstest2016 | -|-|-|-|-| -| Base | 26.35 | 29.07 | 33.30 | -| Big | 27.07 | 30.09 | 34.38 | - -我们这里也提供了以上 [base model](https://transformer-res.bj.bcebos.com/base_model.tar.gz) 和 [big model](https://transformer-res.bj.bcebos.com/big_model.tar.gz) 模型的下载以供使用。 - -### 分布式训练 - -Transformer 模型支持同步或者异步的分布式训练。分布式的配置主要两个方面: - -1 命令行配置 - - - `--local`，有两个取值，`True`表示单机训练，而`False`表示使用分布式训练。默认为单机训练模式。 - - - `--sync`，有两个取值，但只有当`--local`参数为False才会产生影响，其中`True`表示同步训练模式，`False`表示异步训练模式。默认为同步训练模式。 - -2 环境变量配置 - - 在分布式训练模式下，会手动配置训练的trainer数量和pserver数量。在网络拓扑上，每一个trainer都会和每一个pserver相连，pserver作为服务端，而trainer作为客户端。下面分pserver和trainer说明具体的参数配置： - -1) pserver配置 - -- `PADDLE_IS_LOCAL=[0|1]` 是否是分布式训练，`0`标识是分布式，`1`标识是单机 - -- `TRAINING_ROLE=PSERVER` 标识当前节点是pserver - -- `POD_IP=ip` 设置当前pserver使用对外服务的地址 - -- `PADDLE_PORT=port` 设置当前pserver对外服务监听端口号，和`POD_IP`共同构成对外的唯一标识 - -- `PADDLE_TRAINERS_NUM=num` 设置pserver连接的trainer的数量 - -下面是配置的示例, 使用两个pserver, 192.168.2.2上的配置如下: -``` -export PADDLE_PSERVERS=192.168.2.2,192.168.2.3 -export POD_IP=192.168.2.2 -export PADDLE_TRAINERS_NUM=2 -export TRAINING_ROLE=PSERVER -export PADDLE_IS_LOCAL=0 -export PADDLE_PORT=6177 -``` -192.168.2.3上的配置如下: -``` -export PADDLE_PSERVERS=192.168.2.2,192.168.2.3 -export POD_IP=192.168.2.3 -export PADDLE_TRAINERS_NUM=2 -export TRAINING_ROLE=PSERVER -export PADDLE_IS_LOCAL=0 -export PADDLE_PORT=6177 -``` -2) trainer配置 - -- `PADDLE_IS_LOCAL=[0|1]` 是否是分布式训练，`0`标识是分布式，`1`标识是单机 - -- `TRAINING_ROLE=TRAINER` 标识当前节点是trainer - -- `PADDLE_PSERVERS=[ip1,ip2,……]` 设置pserver的ip地址,用于告知trainer互联的pserver的ip, 使用`,`分割 - -- `PADDLE_TRAINER_ID=num` 设置当前节点的编号, 编号的取值范围为0到N-1的整数 - -- `PADDLE_PORT=port` 设置请求的pserver服务端口号 - -下面是配置的示例, 使用两个trainer, trainer 1上的配置如下: -``` -export TRAINING_ROLE=TRAINER -export PADDLE_PSERVERS=192.168.2.2,192.168.2.3 -export PADDLE_TRAINERS_NUM=2 -export PADDLE_TRAINER_ID=0 -export PADDLE_IS_LOCAL=0 -export PADDLE_PORT=6177 -``` -trainer 2上的配置如下: -``` -export TRAINING_ROLE=TRAINER -export PADDLE_PSERVERS=192.168.2.2,192.168.2.3 -export PADDLE_TRAINERS_NUM=2 -export PADDLE_TRAINER_ID=1 -export PADDLE_IS_LOCAL=0 -export PADDLE_PORT=6177 -``` - -### 参考文献 -1. Vaswani A, Shazeer N, Parmar N, et al. [Attention is all you need](http://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf)[C]//Advances in Neural Information Processing Systems. 2017: 6000-6010. -2. He K, Zhang X, Ren S, et al. [Deep residual learning for image recognition](http://openaccess.thecvf.com/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf)[C]//Proceedings of the IEEE conference on computer vision and pattern recognition. 2016: 770-778. -3. Ba J L, Kiros J R, Hinton G E. [Layer normalization](https://arxiv.org/pdf/1607.06450.pdf)[J]. arXiv preprint arXiv:1607.06450, 2016. -4. Sennrich R, Haddow B, Birch A. [Neural machine translation of rare words with subword units](https://arxiv.org/pdf/1508.07909)[J]. arXiv preprint arXiv:1508.07909, 2015. diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/_ce.py b/PaddleNLP/unarchived/neural_machine_translation/transformer/_ce.py deleted file mode 100644 index 447652c4f4d60765011a621371b381e75573612e..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/_ce.py +++ /dev/null @@ -1,68 +0,0 @@ -####this file is only used for continuous evaluation test! - -import os -import sys -sys.path.insert(0, os.environ['ceroot']) -from kpi import CostKpi, DurationKpi, AccKpi - -#### NOTE kpi.py should shared in models in some way!!!! - -train_cost_card1_kpi = CostKpi('train_cost_card1', 0.02, 0, actived=True) -test_cost_card1_kpi = CostKpi('test_cost_card1', 0.005, 0, actived=True) -train_duration_card1_kpi = DurationKpi( - 'train_duration_card1', 0.06, 0, actived=True) -train_cost_card4_kpi = CostKpi('train_cost_card4', 0.01, 0, actived=True) -test_cost_card4_kpi = CostKpi('test_cost_card4', 0.005, 0, actived=True) -train_duration_card4_kpi = DurationKpi( - 'train_duration_card4', 0.06, 0, actived=True) - -tracking_kpis = [ - train_cost_card1_kpi, - test_cost_card1_kpi, - train_duration_card1_kpi, - train_cost_card4_kpi, - test_cost_card4_kpi, - train_duration_card4_kpi, -] - - -def parse_log(log): - ''' - This method should be implemented by model developers. - The suggestion: - each line in the log should be key, value, for example: - " - train_cost\t1.0 - test_cost\t1.0 - train_cost\t1.0 - train_cost\t1.0 - train_acc\t1.2 - " - ''' - for line in log.split('\n'): - fs = line.strip().split('\t') - print(fs) - if len(fs) == 3 and fs[0] == 'kpis': - print("-----%s" % fs) - kpi_name = fs[1] - kpi_value = float(fs[2]) - yield kpi_name, kpi_value - - -def log_to_ce(log): - kpi_tracker = {} - for kpi in tracking_kpis: - kpi_tracker[kpi.name] = kpi - - for (kpi_name, kpi_value) in parse_log(log): - print(kpi_name, kpi_value) - kpi_tracker[kpi_name].add_record(kpi_value) - kpi_tracker[kpi_name].persist() - - -if __name__ == '__main__': - log = sys.stdin.read() - print("*****") - print(log) - print("****") - log_to_ce(log) \ No newline at end of file diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/config.py b/PaddleNLP/unarchived/neural_machine_translation/transformer/config.py deleted file mode 100644 index 0be63dee020a26dea9e6faa1fa02e2fd0b573c64..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/config.py +++ /dev/null @@ -1,214 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -class TrainTaskConfig(object): - # support both CPU and GPU now. - use_gpu = True - # the epoch number to train. - pass_num = 30 - # the number of sequences contained in a mini-batch. - # deprecated, set batch_size in args. - batch_size = 32 - # the hyper parameters for Adam optimizer. - # This static learning_rate will be multiplied to the LearningRateScheduler - # derived learning rate the to get the final learning rate. - learning_rate = 2.0 - beta1 = 0.9 - beta2 = 0.997 - eps = 1e-9 - # the parameters for learning rate scheduling. - warmup_steps = 8000 - # the weight used to mix up the ground-truth distribution and the fixed - # uniform distribution in label smoothing when training. - # Set this as zero if label smoothing is not wanted. - label_smooth_eps = 0.1 - # the directory for saving trained models. - model_dir = "trained_models" - # the directory for saving checkpoints. - ckpt_dir = "trained_ckpts" - # the directory for loading checkpoint. - # If provided, continue training from the checkpoint. - ckpt_path = None - # the parameter to initialize the learning rate scheduler. - # It should be provided if use checkpoints, since the checkpoint doesn't - # include the training step counter currently. - start_step = 0 - # the frequency to save trained models. - save_freq = 10000 - - -class InferTaskConfig(object): - use_gpu = True - # the number of examples in one run for sequence generation. - batch_size = 10 - # the parameters for beam search. - beam_size = 5 - max_out_len = 256 - # the number of decoded sentences to output. - n_best = 1 - # the flags indicating whether to output the special tokens. - output_bos = False - output_eos = False - output_unk = True - # the directory for loading the trained model. - model_path = "trained_models/pass_1.infer.model" - - -class ModelHyperParams(object): - # These following five vocabularies related configurations will be set - # automatically according to the passed vocabulary path and special tokens. - # size of source word dictionary. - src_vocab_size = 10000 - # size of target word dictionay - trg_vocab_size = 10000 - # index for token - bos_idx = 0 - # index for token - eos_idx = 1 - # index for token - unk_idx = 2 - # max length of sequences deciding the size of position encoding table. - max_length = 256 - # the dimension for word embeddings, which is also the last dimension of - # the input and output of multi-head attention, position-wise feed-forward - # networks, encoder and decoder. - d_model = 512 - # size of the hidden layer in position-wise feed-forward networks. - d_inner_hid = 2048 - # the dimension that keys are projected to for dot-product attention. - d_key = 64 - # the dimension that values are projected to for dot-product attention. - d_value = 64 - # number of head used in multi-head attention. - n_head = 8 - # number of sub-layers to be stacked in the encoder and decoder. - n_layer = 6 - # dropout rates of different modules. - prepostprocess_dropout = 0.1 - attention_dropout = 0.1 - relu_dropout = 0.1 - # to process before each sub-layer - preprocess_cmd = "n" # layer normalization - # to process after each sub-layer - postprocess_cmd = "da" # dropout + residual connection - # random seed used in dropout for CE. - dropout_seed = None - # the flag indicating whether to share embedding and softmax weights. - # vocabularies in source and target should be same for weight sharing. - weight_sharing = True - - -def merge_cfg_from_list(cfg_list, g_cfgs): - """ - Set the above global configurations using the cfg_list. - """ - assert len(cfg_list) % 2 == 0 - for key, value in zip(cfg_list[0::2], cfg_list[1::2]): - for g_cfg in g_cfgs: - if hasattr(g_cfg, key): - try: - value = eval(value) - except Exception: # for file path - pass - setattr(g_cfg, key, value) - break - - -# The placeholder for batch_size in compile time. Must be -1 currently to be -# consistent with some ops' infer-shape output in compile time, such as the -# sequence_expand op used in beamsearch decoder. -batch_size = -1 -# The placeholder for squence length in compile time. -seq_len = ModelHyperParams.max_length -# Here list the data shapes and data types of all inputs. -# The shapes here act as placeholder and are set to pass the infer-shape in -# compile time. -input_descs = { - # The actual data shape of src_word is: - # [batch_size, max_src_len_in_batch, 1] - "src_word": [(batch_size, seq_len, 1), "int64", 2], - # The actual data shape of src_pos is: - # [batch_size, max_src_len_in_batch, 1] - "src_pos": [(batch_size, seq_len, 1), "int64"], - # This input is used to remove attention weights on paddings in the - # encoder. - # The actual data shape of src_slf_attn_bias is: - # [batch_size, n_head, max_src_len_in_batch, max_src_len_in_batch] - "src_slf_attn_bias": [(batch_size, ModelHyperParams.n_head, seq_len, - seq_len), "float32"], - # The actual data shape of trg_word is: - # [batch_size, max_trg_len_in_batch, 1] - "trg_word": [(batch_size, seq_len, 1), "int64", - 2], # lod_level is only used in fast decoder. - # The actual data shape of trg_pos is: - # [batch_size, max_trg_len_in_batch, 1] - "trg_pos": [(batch_size, seq_len, 1), "int64"], - # This input is used to remove attention weights on paddings and - # subsequent words in the decoder. - # The actual data shape of trg_slf_attn_bias is: - # [batch_size, n_head, max_trg_len_in_batch, max_trg_len_in_batch] - "trg_slf_attn_bias": [(batch_size, ModelHyperParams.n_head, seq_len, - seq_len), "float32"], - # This input is used to remove attention weights on paddings of the source - # input in the encoder-decoder attention. - # The actual data shape of trg_src_attn_bias is: - # [batch_size, n_head, max_trg_len_in_batch, max_src_len_in_batch] - "trg_src_attn_bias": [(batch_size, ModelHyperParams.n_head, seq_len, - seq_len), "float32"], - # This input is used in independent decoder program for inference. - # The actual data shape of enc_output is: - # [batch_size, max_src_len_in_batch, d_model] - "enc_output": [(batch_size, seq_len, ModelHyperParams.d_model), "float32"], - # The actual data shape of label_word is: - # [batch_size * max_trg_len_in_batch, 1] - "lbl_word": [(batch_size * seq_len, 1), "int64"], - # This input is used to mask out the loss of paddding tokens. - # The actual data shape of label_weight is: - # [batch_size * max_trg_len_in_batch, 1] - "lbl_weight": [(batch_size * seq_len, 1), "float32"], - # This input is used in beam-search decoder. - "init_score": [(batch_size, 1), "float32", 2], - # This input is used in beam-search decoder for the first gather - # (cell states updation) - "init_idx": [(batch_size, ), "int32"], -} - -# Names of word embedding table which might be reused for weight sharing. -word_emb_param_names = ( - "src_word_emb_table", - "trg_word_emb_table", ) -# Names of position encoding table which will be initialized externally. -pos_enc_param_names = ( - "src_pos_enc_table", - "trg_pos_enc_table", ) -# separated inputs for different usages. -encoder_data_input_fields = ( - "src_word", - "src_pos", - "src_slf_attn_bias", ) -decoder_data_input_fields = ( - "trg_word", - "trg_pos", - "trg_slf_attn_bias", - "trg_src_attn_bias", - "enc_output", ) -label_data_input_fields = ( - "lbl_word", - "lbl_weight", ) -# In fast decoder, trg_pos (only containing the current time step) is generated -# by ops and trg_slf_attn_bias is not needed. -fast_decoder_data_input_fields = ( - "trg_word", - "init_score", - "init_idx", - "trg_src_attn_bias", ) diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/gen_data.sh b/PaddleNLP/unarchived/neural_machine_translation/transformer/gen_data.sh deleted file mode 100644 index e00ae05d9c5cc59b7b401428f6e1252397debfe9..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/gen_data.sh +++ /dev/null @@ -1,220 +0,0 @@ -#! /usr/bin/env bash - -set -e - -OUTPUT_DIR=$PWD/gen_data - -############################################################################### -# change these variables for other WMT data -############################################################################### -OUTPUT_DIR_DATA="${OUTPUT_DIR}/wmt16_ende_data" -OUTPUT_DIR_BPE_DATA="${OUTPUT_DIR}/wmt16_ende_data_bpe" -LANG1="en" -LANG2="de" -# each of TRAIN_DATA: data_url data_file_lang1 data_file_lang2 -TRAIN_DATA=( -'http://www.statmt.org/europarl/v7/de-en.tgz' -'europarl-v7.de-en.en' 'europarl-v7.de-en.de' -'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz' -'commoncrawl.de-en.en' 'commoncrawl.de-en.de' -'http://data.statmt.org/wmt16/translation-task/training-parallel-nc-v11.tgz' -'news-commentary-v11.de-en.en' 'news-commentary-v11.de-en.de' -) -# each of DEV_TEST_DATA: data_url data_file_lang1 data_file_lang2 -DEV_TEST_DATA=( -'http://data.statmt.org/wmt16/translation-task/dev.tgz' -'newstest201[45]-deen-ref.en.sgm' 'newstest201[45]-deen-src.de.sgm' -'http://data.statmt.org/wmt16/translation-task/test.tgz' -'newstest2016-deen-ref.en.sgm' 'newstest2016-deen-src.de.sgm' -) -############################################################################### - -############################################################################### -# change these variables for other WMT data -############################################################################### -# OUTPUT_DIR_DATA="${OUTPUT_DIR}/wmt14_enfr_data" -# OUTPUT_DIR_BPE_DATA="${OUTPUT_DIR}/wmt14_enfr_data_bpe" -# LANG1="en" -# LANG2="fr" -# # each of TRAIN_DATA: ata_url data_tgz data_file -# TRAIN_DATA=( -# 'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz' -# 'commoncrawl.fr-en.en' 'commoncrawl.fr-en.fr' -# 'http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz' -# 'training/europarl-v7.fr-en.en' 'training/europarl-v7.fr-en.fr' -# 'http://www.statmt.org/wmt14/training-parallel-nc-v9.tgz' -# 'training/news-commentary-v9.fr-en.en' 'training/news-commentary-v9.fr-en.fr' -# 'http://www.statmt.org/wmt10/training-giga-fren.tar' -# 'giga-fren.release2.fixed.en.*' 'giga-fren.release2.fixed.fr.*' -# 'http://www.statmt.org/wmt13/training-parallel-un.tgz' -# 'un/undoc.2000.fr-en.en' 'un/undoc.2000.fr-en.fr' -# ) -# # each of DEV_TEST_DATA: data_url data_tgz data_file_lang1 data_file_lang2 -# DEV_TEST_DATA=( -# 'http://data.statmt.org/wmt16/translation-task/dev.tgz' -# '.*/newstest201[45]-fren-ref.en.sgm' '.*/newstest201[45]-fren-src.fr.sgm' -# 'http://data.statmt.org/wmt16/translation-task/test.tgz' -# '.*/newstest2016-fren-ref.en.sgm' '.*/newstest2016-fren-src.fr.sgm' -# ) -############################################################################### - -mkdir -p $OUTPUT_DIR_DATA $OUTPUT_DIR_BPE_DATA - -# Extract training data -for ((i=0;i<${#TRAIN_DATA[@]};i+=3)); do - data_url=${TRAIN_DATA[i]} - data_tgz=${data_url##*/} # training-parallel-commoncrawl.tgz - data=${data_tgz%.*} # training-parallel-commoncrawl - data_lang1=${TRAIN_DATA[i+1]} - data_lang2=${TRAIN_DATA[i+2]} - if [ ! -e ${OUTPUT_DIR_DATA}/${data_tgz} ]; then - echo "Download "${data_url} - wget -O ${OUTPUT_DIR_DATA}/${data_tgz} ${data_url} - fi - - if [ ! -d ${OUTPUT_DIR_DATA}/${data} ]; then - echo "Extract "${data_tgz} - mkdir -p ${OUTPUT_DIR_DATA}/${data} - tar_type=${data_tgz:0-3} - if [ ${tar_type} == "tar" ]; then - tar -xvf ${OUTPUT_DIR_DATA}/${data_tgz} -C ${OUTPUT_DIR_DATA}/${data} - else - tar -xvzf ${OUTPUT_DIR_DATA}/${data_tgz} -C ${OUTPUT_DIR_DATA}/${data} - fi - fi - # concatenate all training data - for data_lang in $data_lang1 $data_lang2; do - for f in `find ${OUTPUT_DIR_DATA}/${data} -regex ".*/${data_lang}"`; do - data_dir=`dirname $f` - data_file=`basename $f` - f_base=${f%.*} - f_ext=${f##*.} - if [ $f_ext == "gz" ]; then - gunzip $f - l=${f_base##*.} - f_base=${f_base%.*} - else - l=${f_ext} - fi - - if [ $i -eq 0 ]; then - cat ${f_base}.$l > ${OUTPUT_DIR_DATA}/train.$l - else - cat ${f_base}.$l >> ${OUTPUT_DIR_DATA}/train.$l - fi - done - done -done - -# Clone mosesdecoder -if [ ! -d ${OUTPUT_DIR}/mosesdecoder ]; then - echo "Cloning moses for data processing" - git clone https://github.com/moses-smt/mosesdecoder.git ${OUTPUT_DIR}/mosesdecoder -fi - -# Extract develop and test data -dev_test_data="" -for ((i=0;i<${#DEV_TEST_DATA[@]};i+=3)); do - data_url=${DEV_TEST_DATA[i]} - data_tgz=${data_url##*/} # training-parallel-commoncrawl.tgz - data=${data_tgz%.*} # training-parallel-commoncrawl - data_lang1=${DEV_TEST_DATA[i+1]} - data_lang2=${DEV_TEST_DATA[i+2]} - if [ ! -e ${OUTPUT_DIR_DATA}/${data_tgz} ]; then - echo "Download "${data_url} - wget -O ${OUTPUT_DIR_DATA}/${data_tgz} ${data_url} - fi - - if [ ! -d ${OUTPUT_DIR_DATA}/${data} ]; then - echo "Extract "${data_tgz} - mkdir -p ${OUTPUT_DIR_DATA}/${data} - tar_type=${data_tgz:0-3} - if [ ${tar_type} == "tar" ]; then - tar -xvf ${OUTPUT_DIR_DATA}/${data_tgz} -C ${OUTPUT_DIR_DATA}/${data} - else - tar -xvzf ${OUTPUT_DIR_DATA}/${data_tgz} -C ${OUTPUT_DIR_DATA}/${data} - fi - fi - - for data_lang in $data_lang1 $data_lang2; do - for f in `find ${OUTPUT_DIR_DATA}/${data} -regex ".*/${data_lang}"`; do - data_dir=`dirname $f` - data_file=`basename $f` - data_out=`echo ${data_file} | cut -d '-' -f 1` # newstest2016 - l=`echo ${data_file} | cut -d '.' -f 2` # en - dev_test_data="${dev_test_data}\|${data_out}" # to make regexp - if [ ! -e ${OUTPUT_DIR_DATA}/${data_out}.$l ]; then - ${OUTPUT_DIR}/mosesdecoder/scripts/ems/support/input-from-sgm.perl \ - < $f > ${OUTPUT_DIR_DATA}/${data_out}.$l - fi - done - done -done - -# Tokenize data -for l in ${LANG1} ${LANG2}; do - for f in `ls ${OUTPUT_DIR_DATA}/*.$l | grep "\(train${dev_test_data}\)\.$l$"`; do - f_base=${f%.*} # dir/train dir/newstest2016 - f_out=$f_base.tok.$l - if [ ! -e $f_out ]; then - echo "Tokenize "$f - ${OUTPUT_DIR}/mosesdecoder/scripts/tokenizer/tokenizer.perl -q -l $l -threads 8 < $f > $f_out - fi - done -done - -# Clean data -for f in ${OUTPUT_DIR_DATA}/train.${LANG1} ${OUTPUT_DIR_DATA}/train.tok.${LANG1}; do - f_base=${f%.*} # dir/train dir/train.tok - f_out=${f_base}.clean - if [ ! -e $f_out.${LANG1} ] && [ ! -e $f_out.${LANG2} ]; then - echo "Clean "${f_base} - ${OUTPUT_DIR}/mosesdecoder/scripts/training/clean-corpus-n.perl $f_base ${LANG1} ${LANG2} ${f_out} 1 80 - fi -done - -# Clone subword-nmt and generate BPE data -if [ ! -d ${OUTPUT_DIR}/subword-nmt ]; then - git clone https://github.com/rsennrich/subword-nmt.git ${OUTPUT_DIR}/subword-nmt -fi - -# Generate BPE data and vocabulary -for num_operations in 32000; do - if [ ! -e ${OUTPUT_DIR_BPE_DATA}/bpe.${num_operations} ]; then - echo "Learn BPE with ${num_operations} merge operations" - cat ${OUTPUT_DIR_DATA}/train.tok.clean.${LANG1} ${OUTPUT_DIR_DATA}/train.tok.clean.${LANG2} | \ - ${OUTPUT_DIR}/subword-nmt/learn_bpe.py -s $num_operations > ${OUTPUT_DIR_BPE_DATA}/bpe.${num_operations} - fi - - for l in ${LANG1} ${LANG2}; do - for f in `ls ${OUTPUT_DIR_DATA}/*.$l | grep "\(train${dev_test_data}\)\.tok\(\.clean\)\?\.$l$"`; do - f_base=${f%.*} # dir/train.tok dir/train.tok.clean dir/newstest2016.tok - f_base=${f_base##*/} # train.tok train.tok.clean newstest2016.tok - f_out=${OUTPUT_DIR_BPE_DATA}/${f_base}.bpe.${num_operations}.$l - if [ ! -e $f_out ]; then - echo "Apply BPE to "$f - ${OUTPUT_DIR}/subword-nmt/apply_bpe.py -c ${OUTPUT_DIR_BPE_DATA}/bpe.${num_operations} < $f > $f_out - fi - done - done - - if [ ! -e ${OUTPUT_DIR_BPE_DATA}/vocab.bpe.${num_operations} ]; then - echo "Create vocabulary for BPE data" - cat ${OUTPUT_DIR_BPE_DATA}/train.tok.clean.bpe.${num_operations}.${LANG1} ${OUTPUT_DIR_BPE_DATA}/train.tok.clean.bpe.${num_operations}.${LANG2} | \ - ${OUTPUT_DIR}/subword-nmt/get_vocab.py | cut -f1 -d ' ' > ${OUTPUT_DIR_BPE_DATA}/vocab.bpe.${num_operations} - fi -done - -# Adapt to the reader -for f in ${OUTPUT_DIR_BPE_DATA}/*.bpe.${num_operations}.${LANG1}; do - f_base=${f%.*} # dir/train.tok.clean.bpe.32000 dir/newstest2016.tok.bpe.32000 - f_out=${f_base}.${LANG1}-${LANG2} - if [ ! -e $f_out ]; then - paste -d '\t' $f_base.${LANG1} $f_base.${LANG2} > $f_out - fi -done -if [ ! -e ${OUTPUT_DIR_BPE_DATA}/vocab_all.bpe.${num_operations} ]; then - sed '1i\\n\n' ${OUTPUT_DIR_BPE_DATA}/vocab.bpe.${num_operations} > ${OUTPUT_DIR_BPE_DATA}/vocab_all.bpe.${num_operations} -fi - -echo "All done." diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/images/attention_formula.png b/PaddleNLP/unarchived/neural_machine_translation/transformer/images/attention_formula.png deleted file mode 100644 index 249857f524b4137bafc2d4d1b779ed62d1437b6d..0000000000000000000000000000000000000000 Binary files a/PaddleNLP/unarchived/neural_machine_translation/transformer/images/attention_formula.png and /dev/null differ diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/images/multi_head_attention.png b/PaddleNLP/unarchived/neural_machine_translation/transformer/images/multi_head_attention.png deleted file mode 100644 index 427fb6b32aaeb7013066a167aab4fb97c024c2d6..0000000000000000000000000000000000000000 Binary files a/PaddleNLP/unarchived/neural_machine_translation/transformer/images/multi_head_attention.png and /dev/null differ diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/images/transformer_network.png b/PaddleNLP/unarchived/neural_machine_translation/transformer/images/transformer_network.png deleted file mode 100644 index 34be0e5c7e2b08f858683d86353db5e81049c7ca..0000000000000000000000000000000000000000 Binary files a/PaddleNLP/unarchived/neural_machine_translation/transformer/images/transformer_network.png and /dev/null differ diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/infer.py b/PaddleNLP/unarchived/neural_machine_translation/transformer/infer.py deleted file mode 100644 index 9c4246922c3d9daa91039c438beff2657aaab744..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/infer.py +++ /dev/null @@ -1,329 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import ast -import multiprocessing -import numpy as np -import os -from functools import partial - -import paddle -import paddle.fluid as fluid - -import model -import reader -from config import * -from model import wrap_encoder as encoder -from model import wrap_decoder as decoder -from model import fast_decode as fast_decoder -from train import pad_batch_data, prepare_data_generator - - -def parse_args(): - parser = argparse.ArgumentParser("Training for Transformer.") - parser.add_argument( - "--src_vocab_fpath", - type=str, - required=True, - help="The path of vocabulary file of source language.") - parser.add_argument( - "--trg_vocab_fpath", - type=str, - required=True, - help="The path of vocabulary file of target language.") - parser.add_argument( - "--test_file_pattern", - type=str, - required=True, - help="The pattern to match test data files.") - parser.add_argument( - "--batch_size", - type=int, - default=50, - help="The number of examples in one run for sequence generation.") - parser.add_argument( - "--pool_size", - type=int, - default=10000, - help="The buffer size to pool data.") - parser.add_argument( - "--special_token", - type=str, - default=["", "", ""], - nargs=3, - help="The , and tokens in the dictionary.") - parser.add_argument( - "--token_delimiter", - type=lambda x: str(x.encode().decode("unicode-escape")), - default=" ", - help="The delimiter used to split tokens in source or target sentences. " - "For EN-DE BPE data we provided, use spaces as token delimiter. ") - parser.add_argument( - "--use_py_reader", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to use py_reader.") - parser.add_argument( - "--use_parallel_exe", - type=ast.literal_eval, - default=False, - help="The flag indicating whether to use ParallelExecutor.") - parser.add_argument( - 'opts', - help='See config.py for all options', - default=None, - nargs=argparse.REMAINDER) - args = parser.parse_args() - # Append args related to dict - src_dict = reader.DataReader.load_dict(args.src_vocab_fpath) - trg_dict = reader.DataReader.load_dict(args.trg_vocab_fpath) - dict_args = [ - "src_vocab_size", str(len(src_dict)), "trg_vocab_size", - str(len(trg_dict)), "bos_idx", str(src_dict[args.special_token[0]]), - "eos_idx", str(src_dict[args.special_token[1]]), "unk_idx", - str(src_dict[args.special_token[2]]) - ] - merge_cfg_from_list(args.opts + dict_args, - [InferTaskConfig, ModelHyperParams]) - return args - - -def post_process_seq(seq, - bos_idx=ModelHyperParams.bos_idx, - eos_idx=ModelHyperParams.eos_idx, - output_bos=InferTaskConfig.output_bos, - output_eos=InferTaskConfig.output_eos): - """ - Post-process the beam-search decoded sequence. Truncate from the first - and remove the and tokens currently. - """ - eos_pos = len(seq) - 1 - for i, idx in enumerate(seq): - if idx == eos_idx: - eos_pos = i - break - seq = [ - idx for idx in seq[:eos_pos + 1] - if (output_bos or idx != bos_idx) and (output_eos or idx != eos_idx) - ] - return seq - - -def prepare_batch_input(insts, data_input_names, src_pad_idx, bos_idx, n_head, - d_model, place): - """ - Put all padded data needed by beam search decoder into a dict. - """ - src_word, src_pos, src_slf_attn_bias, src_max_len = pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False) - # start tokens - trg_word = np.asarray([[bos_idx]] * len(insts), dtype="int64") - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, 1, 1]).astype("float32") - trg_word = trg_word.reshape(-1, 1, 1) - src_word = src_word.reshape(-1, src_max_len, 1) - src_pos = src_pos.reshape(-1, src_max_len, 1) - - def to_lodtensor(data, place, lod=None): - data_tensor = fluid.LoDTensor() - data_tensor.set(data, place) - if lod is not None: - data_tensor.set_lod(lod) - return data_tensor - - # beamsearch_op must use tensors with lod - init_score = to_lodtensor( - np.zeros_like( - trg_word, dtype="float32").reshape(-1, 1), - place, [range(trg_word.shape[0] + 1)] * 2) - trg_word = to_lodtensor(trg_word, place, [range(trg_word.shape[0] + 1)] * 2) - init_idx = np.asarray(range(len(insts)), dtype="int32") - - data_input_dict = dict( - zip(data_input_names, [ - src_word, src_pos, src_slf_attn_bias, trg_word, init_score, - init_idx, trg_src_attn_bias - ])) - return data_input_dict - - -def prepare_feed_dict_list(data_generator, count, place): - """ - Prepare the list of feed dict for multi-devices. - """ - feed_dict_list = [] - if data_generator is not None: # use_py_reader == False - data_input_names = encoder_data_input_fields + fast_decoder_data_input_fields - data = next(data_generator) - for idx, data_buffer in enumerate(data): - data_input_dict = prepare_batch_input( - data_buffer, data_input_names, ModelHyperParams.eos_idx, - ModelHyperParams.bos_idx, ModelHyperParams.n_head, - ModelHyperParams.d_model, place) - feed_dict_list.append(data_input_dict) - return feed_dict_list if len(feed_dict_list) == count else None - - -def py_reader_provider_wrapper(data_reader, place): - """ - Data provider needed by fluid.layers.py_reader. - """ - - def py_reader_provider(): - data_input_names = encoder_data_input_fields + fast_decoder_data_input_fields - for batch_id, data in enumerate(data_reader()): - data_input_dict = prepare_batch_input( - data, data_input_names, ModelHyperParams.eos_idx, - ModelHyperParams.bos_idx, ModelHyperParams.n_head, - ModelHyperParams.d_model, place) - yield [data_input_dict[item] for item in data_input_names] - - return py_reader_provider - - -def fast_infer(args): - """ - Inference by beam search decoder based solely on Fluid operators. - """ - out_ids, out_scores, pyreader = fast_decoder( - ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - InferTaskConfig.beam_size, - InferTaskConfig.max_out_len, - ModelHyperParams.eos_idx, - use_py_reader=args.use_py_reader) - - # This is used here to set dropout to the test mode. - infer_program = fluid.default_main_program().clone(for_test=True) - - if InferTaskConfig.use_gpu: - place = fluid.CUDAPlace(0) - dev_count = fluid.core.get_cuda_device_count() - else: - place = fluid.CPUPlace() - dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - fluid.io.load_vars( - exe, - InferTaskConfig.model_path, - vars=[ - var for var in infer_program.list_vars() - if isinstance(var, fluid.framework.Parameter) - ]) - - exec_strategy = fluid.ExecutionStrategy() - # For faster executor - exec_strategy.use_experimental_executor = True - exec_strategy.num_threads = 1 - build_strategy = fluid.BuildStrategy() - infer_exe = fluid.ParallelExecutor( - use_cuda=TrainTaskConfig.use_gpu, - main_program=infer_program, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - - # data reader settings for inference - args.train_file_pattern = args.test_file_pattern - args.use_token_batch = False - args.sort_type = reader.SortType.NONE - args.shuffle = False - args.shuffle_batch = False - test_data = prepare_data_generator( - args, - is_test=False, - count=dev_count, - pyreader=pyreader, - py_reader_provider_wrapper=py_reader_provider_wrapper, - place=place) - if args.use_py_reader: - pyreader.start() - data_generator = None - else: - data_generator = test_data() - trg_idx2word = reader.DataReader.load_dict( - dict_path=args.trg_vocab_fpath, reverse=True) - - while True: - try: - feed_dict_list = prepare_feed_dict_list(data_generator, dev_count, - place) - if args.use_parallel_exe: - seq_ids, seq_scores = infer_exe.run( - fetch_list=[out_ids.name, out_scores.name], - feed=feed_dict_list, - return_numpy=False) - else: - seq_ids, seq_scores = exe.run( - program=infer_program, - fetch_list=[out_ids.name, out_scores.name], - feed=feed_dict_list[0] - if feed_dict_list is not None else None, - return_numpy=False, - use_program_cache=False) - seq_ids_list, seq_scores_list = [ - seq_ids - ], [seq_scores] if isinstance( - seq_ids, paddle.fluid.LoDTensor) else (seq_ids, seq_scores) - for seq_ids, seq_scores in zip(seq_ids_list, seq_scores_list): - # How to parse the results: - # Suppose the lod of seq_ids is: - # [[0, 3, 6], [0, 12, 24, 40, 54, 67, 82]] - # then from lod[0]: - # there are 2 source sentences, beam width is 3. - # from lod[1]: - # the first source sentence has 3 hyps; the lengths are 12, 12, 16 - # the second source sentence has 3 hyps; the lengths are 14, 13, 15 - hyps = [[] for i in range(len(seq_ids.lod()[0]) - 1)] - scores = [[] for i in range(len(seq_scores.lod()[0]) - 1)] - for i in range(len(seq_ids.lod()[0]) - - 1): # for each source sentence - start = seq_ids.lod()[0][i] - end = seq_ids.lod()[0][i + 1] - for j in range(end - start): # for each candidate - sub_start = seq_ids.lod()[1][start + j] - sub_end = seq_ids.lod()[1][start + j + 1] - hyps[i].append(" ".join([ - trg_idx2word[idx] - for idx in post_process_seq( - np.array(seq_ids)[sub_start:sub_end]) - ])) - scores[i].append(np.array(seq_scores)[sub_end - 1]) - print(hyps[i][-1]) - if len(hyps[i]) >= InferTaskConfig.n_best: - break - except (StopIteration, fluid.core.EOFException): - # The data pass is over. - if args.use_py_reader: - pyreader.reset() - break - - -if __name__ == "__main__": - args = parse_args() - fast_infer(args) diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/local_dist.sh b/PaddleNLP/unarchived/neural_machine_translation/transformer/local_dist.sh deleted file mode 100755 index 331845f06fcf0f96332eefed33426eb0596d2b31..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/local_dist.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -set -x - -unset http_proxy -unset https_proxy - -#pserver -export TRAINING_ROLE=PSERVER -export PADDLE_PORT=30134 -export PADDLE_PSERVERS=127.0.0.1 -export PADDLE_IS_LOCAL=0 -export PADDLE_INIT_TRAINER_COUNT=1 -export POD_IP=127.0.0.1 -export PADDLE_TRAINER_ID=0 -export PADDLE_TRAINERS_NUM=1 - -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/:/usr/local/lib/:/workspace/brpc -export PYTHONPATH=$PYTHONPATH:/paddle/build/build_reader_RelWithDebInfo_gpu/python - -#GLOG_v=7 GLOG_logtostderr=1 -CUDA_VISIBLE_DEVICES=4,5,6,7 python -u train.py \ - --src_vocab_fpath 'cluster_test_data_en_fr/thirdparty/vocab.wordpiece.en-fr' \ - --trg_vocab_fpath 'cluster_test_data_en_fr/thirdparty/vocab.wordpiece.en-fr' \ - --special_token '' '' '' \ - --token_delimiter '\x01' \ - --train_file_pattern 'cluster_test_data_en_fr/train/train.wordpiece.en-fr.0' \ - --val_file_pattern 'cluster_test_data_en_fr/thirdparty/newstest2014.wordpiece.en-fr' \ - --use_token_batch True \ - --batch_size 3200 \ - --sort_type pool \ - --pool_size 200000 \ - --local False > pserver.log 2>&1 & - -pserver_pid=$(echo $!) -echo $pserver_pid - -sleep 30s - -#trainer -export TRAINING_ROLE=TRAINER -export PADDLE_PORT=30134 -export PADDLE_PSERVERS=127.0.0.1 -export PADDLE_IS_LOCAL=0 -export PADDLE_INIT_TRAINER_COUNT=1 -export POD_IP=127.0.0.1 -export PADDLE_TRAINER_ID=0 -export PADDLE_TRAINERS_NUM=1 - -CUDA_VISIBLE_DEVICES=4,5,6,7 python -u train.py \ - --src_vocab_fpath 'cluster_test_data_en_fr/thirdparty/vocab.wordpiece.en-fr' \ - --trg_vocab_fpath 'cluster_test_data_en_fr/thirdparty/vocab.wordpiece.en-fr' \ - --special_token '' '' '' \ - --token_delimiter '\x01' \ - --train_file_pattern 'cluster_test_data_en_fr/train/train.wordpiece.en-fr.0' \ - --val_file_pattern 'cluster_test_data_en_fr/thirdparty/newstest2014.wordpiece.en-fr' \ - --use_token_batch True \ - --batch_size 3200 \ - --sort_type pool \ - --pool_size 200000 \ - --local False > trainer.log 2>&1 & - -#sleep 80 -#kill -9 $pserver_pid diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/model.py b/PaddleNLP/unarchived/neural_machine_translation/transformer/model.py deleted file mode 100644 index f4e6506ad7611aef22f1d0908dff3eb9a2830cf0..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/model.py +++ /dev/null @@ -1,918 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from functools import partial -import numpy as np - -import paddle.fluid as fluid -import paddle.fluid.layers as layers - -from config import * - - -def wrap_layer_with_block(layer, block_idx): - """ - Make layer define support indicating block, by which we can add layers - to other blocks within current block. This will make it easy to define - cache among while loop. - """ - - class BlockGuard(object): - """ - BlockGuard class. - - BlockGuard class is used to switch to the given block in a program by - using the Python `with` keyword. - """ - - def __init__(self, block_idx=None, main_program=None): - self.main_program = fluid.default_main_program( - ) if main_program is None else main_program - self.old_block_idx = self.main_program.current_block().idx - self.new_block_idx = block_idx - - def __enter__(self): - self.main_program.current_block_idx = self.new_block_idx - - def __exit__(self, exc_type, exc_val, exc_tb): - self.main_program.current_block_idx = self.old_block_idx - if exc_type is not None: - return False # re-raise exception - return True - - def layer_wrapper(*args, **kwargs): - with BlockGuard(block_idx): - return layer(*args, **kwargs) - - return layer_wrapper - - -def position_encoding_init(n_position, d_pos_vec): - """ - Generate the initial values for the sinusoid position encoding table. - """ - channels = d_pos_vec - position = np.arange(n_position) - num_timescales = channels // 2 - log_timescale_increment = (np.log(float(1e4) / float(1)) / - (num_timescales - 1)) - inv_timescales = np.exp(np.arange( - num_timescales)) * -log_timescale_increment - scaled_time = np.expand_dims(position, 1) * np.expand_dims(inv_timescales, - 0) - signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1) - signal = np.pad(signal, [[0, 0], [0, np.mod(channels, 2)]], 'constant') - position_enc = signal - return position_enc.astype("float32") - - -def multi_head_attention(queries, - keys, - values, - attn_bias, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0., - cache=None, - gather_idx=None, - static_kv=False): - """ - Multi-Head Attention. Note that attn_bias is added to the logit before - computing softmax activiation to mask certain selected positions so that - they will not considered in attention weights. - """ - keys = queries if keys is None else keys - values = keys if values is None else values - - if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3): - raise ValueError( - "Inputs: quries, keys and values should all be 3-D tensors.") - - def __compute_qkv(queries, keys, values, n_head, d_key, d_value): - """ - Add linear projection to queries, keys, and values. - """ - q = layers.fc(input=queries, - size=d_key * n_head, - bias_attr=False, - num_flatten_dims=2) - # For encoder-decoder attention in inference, insert the ops and vars - # into global block to use as cache among beam search. - fc_layer = wrap_layer_with_block( - layers.fc, fluid.default_main_program().current_block() - .parent_idx) if cache is not None and static_kv else layers.fc - k = fc_layer( - input=keys, - size=d_key * n_head, - bias_attr=False, - num_flatten_dims=2) - v = fc_layer( - input=values, - size=d_value * n_head, - bias_attr=False, - num_flatten_dims=2) - return q, k, v - - def __split_heads_qkv(queries, keys, values, n_head, d_key, d_value): - """ - Reshape input tensors at the last dimension to split multi-heads - and then transpose. Specifically, transform the input tensor with shape - [bs, max_sequence_length, n_head * hidden_dim] to the output tensor - with shape [bs, n_head, max_sequence_length, hidden_dim]. - """ - # The value 0 in shape attr means copying the corresponding dimension - # size of the input as the output dimension size. - reshaped_q = layers.reshape( - x=queries, shape=[0, 0, n_head, d_key], inplace=True) - # permuate the dimensions into: - # [batch_size, n_head, max_sequence_len, hidden_size_per_head] - q = layers.transpose(x=reshaped_q, perm=[0, 2, 1, 3]) - # For encoder-decoder attention in inference, insert the ops and vars - # into global block to use as cache among beam search. - reshape_layer = wrap_layer_with_block( - layers.reshape, - fluid.default_main_program().current_block() - .parent_idx) if cache is not None and static_kv else layers.reshape - transpose_layer = wrap_layer_with_block( - layers.transpose, - fluid.default_main_program().current_block(). - parent_idx) if cache is not None and static_kv else layers.transpose - reshaped_k = reshape_layer( - x=keys, shape=[0, 0, n_head, d_key], inplace=True) - k = transpose_layer(x=reshaped_k, perm=[0, 2, 1, 3]) - reshaped_v = reshape_layer( - x=values, shape=[0, 0, n_head, d_value], inplace=True) - v = transpose_layer(x=reshaped_v, perm=[0, 2, 1, 3]) - - if cache is not None: # only for faster inference - if static_kv: # For encoder-decoder attention in inference - cache_k, cache_v = cache["static_k"], cache["static_v"] - # To init the static_k and static_v in cache. - # Maybe we can use condition_op(if_else) to do these at the first - # step in while loop to replace these, however it might be less - # efficient. - static_cache_init = wrap_layer_with_block( - layers.assign, - fluid.default_main_program().current_block().parent_idx) - static_cache_init(k, cache_k) - static_cache_init(v, cache_v) - else: # For decoder self-attention in inference - cache_k, cache_v = cache["k"], cache["v"] - # gather cell states corresponding to selected parent - select_k = layers.gather(cache_k, index=gather_idx) - select_v = layers.gather(cache_v, index=gather_idx) - if not static_kv: - # For self attention in inference, use cache and concat time steps. - select_k = layers.concat([select_k, k], axis=2) - select_v = layers.concat([select_v, v], axis=2) - # update cell states(caches) cached in global block - layers.assign(select_k, cache_k) - layers.assign(select_v, cache_v) - return q, select_k, select_v - return q, k, v - - def __combine_heads(x): - """ - Transpose and then reshape the last two dimensions of inpunt tensor x - so that it becomes one dimension, which is reverse to __split_heads. - """ - if len(x.shape) != 4: - raise ValueError("Input(x) should be a 4-D Tensor.") - - trans_x = layers.transpose(x, perm=[0, 2, 1, 3]) - # The value 0 in shape attr means copying the corresponding dimension - # size of the input as the output dimension size. - return layers.reshape( - x=trans_x, - shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], - inplace=True) - - def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate): - """ - Scaled Dot-Product Attention - """ - product = layers.matmul(x=q, y=k, transpose_y=True, alpha=d_key**-0.5) - if attn_bias: - product += attn_bias - weights = layers.softmax(product) - if dropout_rate: - weights = layers.dropout( - weights, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) - out = layers.matmul(weights, v) - return out - - q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value) - q, k, v = __split_heads_qkv(q, k, v, n_head, d_key, d_value) - - ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_model, - dropout_rate) - - out = __combine_heads(ctx_multiheads) - - # Project back to the model size. - proj_out = layers.fc(input=out, - size=d_model, - bias_attr=False, - num_flatten_dims=2) - return proj_out - - -def positionwise_feed_forward(x, d_inner_hid, d_hid, dropout_rate): - """ - Position-wise Feed-Forward Networks. - This module consists of two linear transformations with a ReLU activation - in between, which is applied to each position separately and identically. - """ - hidden = layers.fc(input=x, - size=d_inner_hid, - num_flatten_dims=2, - act="relu") - if dropout_rate: - hidden = layers.dropout( - hidden, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) - out = layers.fc(input=hidden, size=d_hid, num_flatten_dims=2) - return out - - -def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.): - """ - Add residual connection, layer normalization and droput to the out tensor - optionally according to the value of process_cmd. - This will be used before or after multi-head attention and position-wise - feed-forward networks. - """ - for cmd in process_cmd: - if cmd == "a": # add residual connection - out = out + prev_out if prev_out else out - elif cmd == "n": # add layer normalization - out = layers.layer_norm( - out, - begin_norm_axis=len(out.shape) - 1, - param_attr=fluid.initializer.Constant(1.), - bias_attr=fluid.initializer.Constant(0.)) - elif cmd == "d": # add dropout - if dropout_rate: - out = layers.dropout( - out, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) - return out - - -pre_process_layer = partial(pre_post_process_layer, None) -post_process_layer = pre_post_process_layer - - -def prepare_encoder_decoder(src_word, - src_pos, - src_vocab_size, - src_emb_dim, - src_max_len, - dropout_rate=0., - word_emb_param_name=None, - pos_enc_param_name=None): - """Add word embeddings and position encodings. - The output tensor has a shape of: - [batch_size, max_src_length_in_batch, d_model]. - This module is used at the bottom of the encoder stacks. - """ - src_word_emb = layers.embedding( - src_word, - size=[src_vocab_size, src_emb_dim], - padding_idx=ModelHyperParams.bos_idx, # set embedding of bos to 0 - param_attr=fluid.ParamAttr( - name=word_emb_param_name, - initializer=fluid.initializer.Normal(0., src_emb_dim**-0.5))) - - src_word_emb = layers.scale(x=src_word_emb, scale=src_emb_dim**0.5) - src_pos_enc = layers.embedding( - src_pos, - size=[src_max_len, src_emb_dim], - param_attr=fluid.ParamAttr( - name=pos_enc_param_name, trainable=False)) - src_pos_enc.stop_gradient = True - enc_input = src_word_emb + src_pos_enc - return layers.dropout( - enc_input, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) if dropout_rate else enc_input - - -prepare_encoder = partial( - prepare_encoder_decoder, pos_enc_param_name=pos_enc_param_names[0]) -prepare_decoder = partial( - prepare_encoder_decoder, pos_enc_param_name=pos_enc_param_names[1]) - - -def encoder_layer(enc_input, - attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): - """The encoder layers that can be stacked to form a deep encoder. - This module consits of a multi-head (self) attention followed by - position-wise feed-forward networks and both the two components companied - with the post_process_layer to add residual connection, layer normalization - and droput. - """ - attn_output = multi_head_attention( - pre_process_layer(enc_input, preprocess_cmd, - prepostprocess_dropout), None, None, attn_bias, d_key, - d_value, d_model, n_head, attention_dropout) - attn_output = post_process_layer(enc_input, attn_output, postprocess_cmd, - prepostprocess_dropout) - ffd_output = positionwise_feed_forward( - pre_process_layer(attn_output, preprocess_cmd, prepostprocess_dropout), - d_inner_hid, d_model, relu_dropout) - return post_process_layer(attn_output, ffd_output, postprocess_cmd, - prepostprocess_dropout) - - -def encoder(enc_input, - attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): - """ - The encoder is composed of a stack of identical layers returned by calling - encoder_layer. - """ - for i in range(n_layer): - enc_output = encoder_layer( - enc_input, - attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, ) - enc_input = enc_output - enc_output = pre_process_layer(enc_output, preprocess_cmd, - prepostprocess_dropout) - return enc_output - - -def decoder_layer(dec_input, - enc_output, - slf_attn_bias, - dec_enc_attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None, - gather_idx=None): - """ The layer to be stacked in decoder part. - The structure of this module is similar to that in the encoder part except - a multi-head attention is added to implement encoder-decoder attention. - """ - slf_attn_output = multi_head_attention( - pre_process_layer(dec_input, preprocess_cmd, prepostprocess_dropout), - None, - None, - slf_attn_bias, - d_key, - d_value, - d_model, - n_head, - attention_dropout, - cache=cache, - gather_idx=gather_idx) - slf_attn_output = post_process_layer( - dec_input, - slf_attn_output, - postprocess_cmd, - prepostprocess_dropout, ) - enc_attn_output = multi_head_attention( - pre_process_layer(slf_attn_output, preprocess_cmd, - prepostprocess_dropout), - enc_output, - enc_output, - dec_enc_attn_bias, - d_key, - d_value, - d_model, - n_head, - attention_dropout, - cache=cache, - gather_idx=gather_idx, - static_kv=True) - enc_attn_output = post_process_layer( - slf_attn_output, - enc_attn_output, - postprocess_cmd, - prepostprocess_dropout, ) - ffd_output = positionwise_feed_forward( - pre_process_layer(enc_attn_output, preprocess_cmd, - prepostprocess_dropout), - d_inner_hid, - d_model, - relu_dropout, ) - dec_output = post_process_layer( - enc_attn_output, - ffd_output, - postprocess_cmd, - prepostprocess_dropout, ) - return dec_output - - -def decoder(dec_input, - enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=None, - gather_idx=None): - """ - The decoder is composed of a stack of identical decoder_layer layers. - """ - for i in range(n_layer): - dec_output = decoder_layer( - dec_input, - enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None if caches is None else caches[i], - gather_idx=gather_idx) - dec_input = dec_output - dec_output = pre_process_layer(dec_output, preprocess_cmd, - prepostprocess_dropout) - return dec_output - - -def make_all_inputs(input_fields): - """ - Define the input data layers for the transformer model. - """ - inputs = [] - for input_field in input_fields: - input_var = layers.data( - name=input_field, - shape=input_descs[input_field][0], - dtype=input_descs[input_field][1], - lod_level=input_descs[input_field][2] - if len(input_descs[input_field]) == 3 else 0, - append_batch_size=False) - inputs.append(input_var) - return inputs - - -def make_all_py_reader_inputs(input_fields, is_test=False): - reader = layers.py_reader( - capacity=20, - name="test_reader" if is_test else "train_reader", - shapes=[input_descs[input_field][0] for input_field in input_fields], - dtypes=[input_descs[input_field][1] for input_field in input_fields], - lod_levels=[ - input_descs[input_field][2] - if len(input_descs[input_field]) == 3 else 0 - for input_field in input_fields - ]) - return layers.read_file(reader), reader - - -def transformer(src_vocab_size, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - label_smooth_eps, - use_py_reader=False, - is_test=False): - if weight_sharing: - assert src_vocab_size == trg_vocab_size, ( - "Vocabularies in source and target should be same for weight sharing." - ) - - data_input_names = encoder_data_input_fields + \ - decoder_data_input_fields[:-1] + label_data_input_fields - - if use_py_reader: - all_inputs, reader = make_all_py_reader_inputs(data_input_names, - is_test) - else: - all_inputs = make_all_inputs(data_input_names) - - enc_inputs_len = len(encoder_data_input_fields) - dec_inputs_len = len(decoder_data_input_fields[:-1]) - enc_inputs = all_inputs[0:enc_inputs_len] - dec_inputs = all_inputs[enc_inputs_len:enc_inputs_len + dec_inputs_len] - label = all_inputs[-2] - weights = all_inputs[-1] - - enc_output = wrap_encoder( - src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - enc_inputs, ) - - predict = wrap_decoder( - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - dec_inputs, - enc_output, ) - - # Padding index do not contribute to the total loss. The weights is used to - # cancel padding index in calculating the loss. - if label_smooth_eps: - label = layers.label_smooth( - label=layers.one_hot( - input=label, depth=trg_vocab_size), - epsilon=label_smooth_eps) - - cost = layers.softmax_with_cross_entropy( - logits=predict, - label=label, - soft_label=True if label_smooth_eps else False) - weighted_cost = cost * weights - sum_cost = layers.reduce_sum(weighted_cost) - token_num = layers.reduce_sum(weights) - token_num.stop_gradient = True - avg_cost = sum_cost / token_num - return sum_cost, avg_cost, predict, token_num, reader if use_py_reader else None - - -def wrap_encoder(src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - enc_inputs=None): - """ - The wrapper assembles together all needed layers for the encoder. - """ - if enc_inputs is None: - # This is used to implement independent encoder program in inference. - src_word, src_pos, src_slf_attn_bias = make_all_inputs( - encoder_data_input_fields) - else: - src_word, src_pos, src_slf_attn_bias = enc_inputs - enc_input = prepare_encoder( - src_word, - src_pos, - src_vocab_size, - d_model, - max_length, - prepostprocess_dropout, - word_emb_param_name=word_emb_param_names[0]) - enc_output = encoder( - enc_input, - src_slf_attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, ) - return enc_output - - -def wrap_decoder(trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - dec_inputs=None, - enc_output=None, - caches=None, - gather_idx=None): - """ - The wrapper assembles together all needed layers for the decoder. - """ - if dec_inputs is None: - # This is used to implement independent decoder program in inference. - trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias, enc_output = \ - make_all_inputs(decoder_data_input_fields) - else: - trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias = dec_inputs - - dec_input = prepare_decoder( - trg_word, - trg_pos, - trg_vocab_size, - d_model, - max_length, - prepostprocess_dropout, - word_emb_param_name=word_emb_param_names[0] - if weight_sharing else word_emb_param_names[1]) - dec_output = decoder( - dec_input, - enc_output, - trg_slf_attn_bias, - trg_src_attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=caches, - gather_idx=gather_idx) - # Reshape to 2D tensor to use GEMM instead of BatchedGEMM - dec_output = layers.reshape( - dec_output, shape=[-1, dec_output.shape[-1]], inplace=True) - if weight_sharing: - predict = layers.matmul( - x=dec_output, - y=fluid.default_main_program().global_block().var( - word_emb_param_names[0]), - transpose_y=True) - else: - predict = layers.fc(input=dec_output, - size=trg_vocab_size, - bias_attr=False) - if dec_inputs is None: - # Return probs for independent decoder program. - predict = layers.softmax(predict) - return predict - - -def fast_decode(src_vocab_size, - trg_vocab_size, - max_in_len, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - beam_size, - max_out_len, - eos_idx, - use_py_reader=False): - """ - Use beam search to decode. Caches will be used to store states of history - steps which can make the decoding faster. - """ - data_input_names = encoder_data_input_fields + fast_decoder_data_input_fields - - if use_py_reader: - all_inputs, reader = make_all_py_reader_inputs(data_input_names) - else: - all_inputs = make_all_inputs(data_input_names) - - enc_inputs_len = len(encoder_data_input_fields) - dec_inputs_len = len(fast_decoder_data_input_fields) - enc_inputs = all_inputs[0:enc_inputs_len] - dec_inputs = all_inputs[enc_inputs_len:enc_inputs_len + dec_inputs_len] - - enc_output = wrap_encoder( - src_vocab_size, - max_in_len, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - enc_inputs, ) - start_tokens, init_scores, parent_idx, trg_src_attn_bias = dec_inputs - - def beam_search(): - max_len = layers.fill_constant( - shape=[1], - dtype=start_tokens.dtype, - value=max_out_len, - force_cpu=True) - step_idx = layers.fill_constant( - shape=[1], dtype=start_tokens.dtype, value=0, force_cpu=True) - cond = layers.less_than(x=step_idx, y=max_len) # default force_cpu=True - while_op = layers.While(cond) - # array states will be stored for each step. - ids = layers.array_write( - layers.reshape(start_tokens, (-1, 1)), step_idx) - scores = layers.array_write(init_scores, step_idx) - # cell states will be overwrited at each step. - # caches contains states of history steps in decoder self-attention - # and static encoder output projections in encoder-decoder attention - # to reduce redundant computation. - caches = [ - { - "k": # for self attention - layers.fill_constant_batch_size_like( - input=start_tokens, - shape=[-1, n_head, 0, d_key], - dtype=enc_output.dtype, - value=0), - "v": # for self attention - layers.fill_constant_batch_size_like( - input=start_tokens, - shape=[-1, n_head, 0, d_value], - dtype=enc_output.dtype, - value=0), - "static_k": # for encoder-decoder attention - layers.create_tensor(dtype=enc_output.dtype), - "static_v": # for encoder-decoder attention - layers.create_tensor(dtype=enc_output.dtype) - } for i in range(n_layer) - ] - - with while_op.block(): - pre_ids = layers.array_read(array=ids, i=step_idx) - # Since beam_search_op dosen't enforce pre_ids' shape, we can do - # inplace reshape here which actually change the shape of pre_ids. - pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True) - pre_scores = layers.array_read(array=scores, i=step_idx) - # gather cell states corresponding to selected parent - pre_src_attn_bias = layers.gather( - trg_src_attn_bias, index=parent_idx) - pre_pos = layers.elementwise_mul( - x=layers.fill_constant_batch_size_like( - input=pre_src_attn_bias, # cann't use lod tensor here - value=1, - shape=[-1, 1, 1], - dtype=pre_ids.dtype), - y=step_idx, - axis=0) - logits = wrap_decoder( - trg_vocab_size, - max_in_len, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - dec_inputs=(pre_ids, pre_pos, None, pre_src_attn_bias), - enc_output=enc_output, - caches=caches, - gather_idx=parent_idx) - # intra-beam topK - topk_scores, topk_indices = layers.topk( - input=layers.softmax(logits), k=beam_size) - accu_scores = layers.elementwise_add( - x=layers.log(topk_scores), y=pre_scores, axis=0) - # beam_search op uses lod to differentiate branches. - accu_scores = layers.lod_reset(accu_scores, pre_ids) - # topK reduction across beams, also contain special handle of - # end beams and end sentences(batch reduction) - selected_ids, selected_scores, gather_idx = layers.beam_search( - pre_ids=pre_ids, - pre_scores=pre_scores, - ids=topk_indices, - scores=accu_scores, - beam_size=beam_size, - end_id=eos_idx, - return_parent_idx=True) - layers.increment(x=step_idx, value=1.0, in_place=True) - # cell states(caches) have been updated in wrap_decoder, - # only need to update beam search states here. - layers.array_write(selected_ids, i=step_idx, array=ids) - layers.array_write(selected_scores, i=step_idx, array=scores) - layers.assign(gather_idx, parent_idx) - layers.assign(pre_src_attn_bias, trg_src_attn_bias) - length_cond = layers.less_than(x=step_idx, y=max_len) - finish_cond = layers.logical_not(layers.is_empty(x=selected_ids)) - layers.logical_and(x=length_cond, y=finish_cond, out=cond) - - finished_ids, finished_scores = layers.beam_search_decode( - ids, scores, beam_size=beam_size, end_id=eos_idx) - return finished_ids, finished_scores - - finished_ids, finished_scores = beam_search() - return finished_ids, finished_scores, reader if use_py_reader else None diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/optim.py b/PaddleNLP/unarchived/neural_machine_translation/transformer/optim.py deleted file mode 100644 index a4d034e9e68b9d9548778d410009a7871a96e9e0..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/optim.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import numpy as np - -import paddle.fluid as fluid -import paddle.fluid.layers as layers - - -class LearningRateScheduler(object): - """ - Wrapper for learning rate scheduling as described in the Transformer paper. - LearningRateScheduler adapts the learning rate externally and the adapted - learning rate will be feeded into the main_program as input data. - """ - - def __init__(self, - d_model, - warmup_steps, - learning_rate=0.001, - current_steps=0, - name="learning_rate"): - self.current_steps = current_steps - self.warmup_steps = warmup_steps - self.d_model = d_model - self.static_lr = learning_rate - self.learning_rate = layers.create_global_var( - name=name, - shape=[1], - value=float(learning_rate), - dtype="float32", - persistable=True) - - def update_learning_rate(self): - self.current_steps += 1 - lr_value = np.power(self.d_model, -0.5) * np.min([ - np.power(self.current_steps, -0.5), - np.power(self.warmup_steps, -1.5) * self.current_steps - ]) * self.static_lr - return np.array([lr_value], dtype="float32") diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/profile.py b/PaddleNLP/unarchived/neural_machine_translation/transformer/profile.py deleted file mode 100644 index d124947b556ce5d0e458aa420fdcb108564a0956..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/profile.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import ast -import contextlib -import multiprocessing -import os -import six -import time - -import numpy as np -import paddle.fluid as fluid -import paddle.fluid.profiler as profiler - -import reader -from config import * -from train import pad_batch_data, prepare_data_generator, \ - prepare_feed_dict_list, py_reader_provider_wrapper -from model import transformer, position_encoding_init - - -def parse_args(): - parser = argparse.ArgumentParser("Training for Transformer.") - parser.add_argument( - "--src_vocab_fpath", - type=str, - required=True, - help="The path of vocabulary file of source language.") - parser.add_argument( - "--trg_vocab_fpath", - type=str, - required=True, - help="The path of vocabulary file of target language.") - parser.add_argument( - "--train_file_pattern", - type=str, - required=True, - help="The pattern to match training data files.") - parser.add_argument( - "--use_token_batch", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to " - "produce batch data according to token number.") - parser.add_argument( - "--batch_size", - type=int, - default=4096, - help="The number of sequences contained in a mini-batch, or the maximum " - "number of tokens (include paddings) contained in a mini-batch. Note " - "that this represents the number on single device and the actual batch " - "size for multi-devices will multiply the device number.") - parser.add_argument( - "--pool_size", - type=int, - default=200000, - help="The buffer size to pool data.") - parser.add_argument( - "--sort_type", - default="pool", - choices=("global", "pool", "none"), - help="The grain to sort by length: global for all instances; pool for " - "instances in pool; none for no sort.") - parser.add_argument( - "--shuffle", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to shuffle instances in each pass.") - parser.add_argument( - "--shuffle_batch", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to shuffle the data batches.") - parser.add_argument( - "--special_token", - type=str, - default=["", "", ""], - nargs=3, - help="The , and tokens in the dictionary.") - parser.add_argument( - "--token_delimiter", - type=lambda x: str(x.encode().decode("unicode-escape")), - default=" ", - help="The delimiter used to split tokens in source or target sentences. " - "For EN-DE BPE data we provided, use spaces as token delimiter.") - parser.add_argument( - "--use_py_reader", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to use py_reader.") - parser.add_argument( - "--iter_num", - type=int, - default=20, - help="The iteration number to run in profiling.") - parser.add_argument( - "--use_parallel_exe", - type=ast.literal_eval, - default=False, - help="The flag indicating whether to use ParallelExecutor.") - parser.add_argument( - "--profile_ops", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to profile operators.") - parser.add_argument( - 'opts', - help='See config.py for all options', - default=None, - nargs=argparse.REMAINDER) - - args = parser.parse_args() - # Append args related to dict - src_dict = reader.DataReader.load_dict(args.src_vocab_fpath) - trg_dict = reader.DataReader.load_dict(args.trg_vocab_fpath) - dict_args = [ - "src_vocab_size", str(len(src_dict)), "trg_vocab_size", - str(len(trg_dict)), "bos_idx", str(src_dict[args.special_token[0]]), - "eos_idx", str(src_dict[args.special_token[1]]), "unk_idx", - str(src_dict[args.special_token[2]]) - ] - merge_cfg_from_list(args.opts + dict_args, - [TrainTaskConfig, ModelHyperParams]) - return args - - -def main(args): - train_prog = fluid.Program() - startup_prog = fluid.Program() - train_prog.random_seed = 1000 - startup_prog.random_seed = 1000 - with fluid.program_guard(train_prog, startup_prog): - with fluid.unique_name.guard(): - sum_cost, avg_cost, predict, token_num, pyreader = transformer( - ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - TrainTaskConfig.label_smooth_eps, - use_py_reader=args.use_py_reader, - is_test=False) - lr_decay = fluid.layers.learning_rate_scheduler.noam_decay( - ModelHyperParams.d_model, TrainTaskConfig.warmup_steps) - optimizer = fluid.optimizer.Adam( - learning_rate=lr_decay * TrainTaskConfig.learning_rate, - beta1=TrainTaskConfig.beta1, - beta2=TrainTaskConfig.beta2, - epsilon=TrainTaskConfig.eps) - optimizer.minimize(avg_cost) - - if TrainTaskConfig.use_gpu: - place = fluid.CUDAPlace(0) - dev_count = fluid.core.get_cuda_device_count() - else: - place = fluid.CPUPlace() - dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - exe = fluid.Executor(place) - # Initialize the parameters. - if TrainTaskConfig.ckpt_path: - fluid.io.load_persistables(exe, TrainTaskConfig.ckpt_path) - else: - exe.run(startup_prog) - - exec_strategy = fluid.ExecutionStrategy() - # For faster executor - exec_strategy.use_experimental_executor = True - exec_strategy.num_iteration_per_drop_scope = 5 - build_strategy = fluid.BuildStrategy() - # Since the token number differs among devices, customize gradient scale to - # use token average cost among multi-devices. and the gradient scale is - # `1 / token_number` for average cost. - # build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized - train_exe = fluid.ParallelExecutor( - use_cuda=TrainTaskConfig.use_gpu, - loss_name=avg_cost.name, - main_program=train_prog, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - - # the best cross-entropy value with label smoothing - loss_normalizer = -((1. - TrainTaskConfig.label_smooth_eps) * np.log( - (1. - TrainTaskConfig.label_smooth_eps - )) + TrainTaskConfig.label_smooth_eps * - np.log(TrainTaskConfig.label_smooth_eps / ( - ModelHyperParams.trg_vocab_size - 1) + 1e-20)) - - train_data = prepare_data_generator( - args, is_test=False, count=dev_count, pyreader=pyreader) - if args.use_py_reader: - pyreader.start() - data_generator = None - else: - data_generator = train_data() - - def run(iter_num): - reader_time = [] - run_time = [] - - for step_idx in six.moves.xrange(iter_num): - try: - start_time = time.time() - feed_dict_list = prepare_feed_dict_list(data_generator, - init_flag, dev_count) - end_time = time.time() - reader_time.append(end_time - start_time) - - start_time = time.time() - if args.use_parallel_exe: - outs = train_exe.run( - fetch_list=[sum_cost.name, token_num.name], - feed=feed_dict_list) - else: - outs = exe.run(program=train_prog, - fetch_list=[sum_cost.name, token_num.name], - feed=feed_dict_list[0] - if feed_dict_list is not None else None) - end_time = time.time() - run_time.append(end_time - start_time) - - sum_cost_val, token_num_val = np.array(outs[0]), np.array(outs[ - 1]) - # sum the cost from multi-devices - total_sum_cost = sum_cost_val.sum() - total_token_num = token_num_val.sum() - total_avg_cost = total_sum_cost / total_token_num - print("step_idx: %d, avg loss: %f, " - "normalized loss: %f, ppl: %f" % - (step_idx, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100)]))) - except (StopIteration, fluid.core.EOFException): - # The current pass is over. - if args.use_py_reader: - pyreader.reset() - pyreader.start() - - return reader_time, run_time - - @contextlib.contextmanager - def profile_context(profile=True): - if profile: - with profiler.profiler('All', 'total', '/tmp/profile_file'): - yield - else: - yield - - # start-up - init_flag = True - run(5) - init_flag = False - - # profiling - start = time.time() - # currently only support profiling on one device - with profile_context(args.profile_ops): - reader_time, run_time = run(args.iter_num) - end = time.time() - total_time = end - start - print( - "Total time: {0}, reader time: {1} s, run time: {2} s, step number: {3}". - format(total_time, np.sum(reader_time), np.sum(run_time), - args.iter_num)) - - -if __name__ == "__main__": - args = parse_args() - main(args) diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/reader.py b/PaddleNLP/unarchived/neural_machine_translation/transformer/reader.py deleted file mode 100644 index 4869213ac05987083b0684fe2c39ac6e22f8aa8a..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/reader.py +++ /dev/null @@ -1,348 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import glob -import six -import os -import tarfile - -import numpy as np - - -class SortType(object): - GLOBAL = 'global' - POOL = 'pool' - NONE = "none" - - -class Converter(object): - def __init__(self, vocab, beg, end, unk, delimiter, add_beg): - self._vocab = vocab - self._beg = beg - self._end = end - self._unk = unk - self._delimiter = delimiter - self._add_beg = add_beg - - def __call__(self, sentence): - return ([self._beg] if self._add_beg else []) + [ - self._vocab.get(w, self._unk) - for w in sentence.split(self._delimiter) - ] + [self._end] - - -class ComposedConverter(object): - def __init__(self, converters): - self._converters = converters - - def __call__(self, parallel_sentence): - return [ - self._converters[i](parallel_sentence[i]) - for i in range(len(self._converters)) - ] - - -class SentenceBatchCreator(object): - def __init__(self, batch_size): - self.batch = [] - self._batch_size = batch_size - - def append(self, info): - self.batch.append(info) - if len(self.batch) == self._batch_size: - tmp = self.batch - self.batch = [] - return tmp - - -class TokenBatchCreator(object): - def __init__(self, batch_size): - self.batch = [] - self.max_len = -1 - self._batch_size = batch_size - - def append(self, info): - cur_len = info.max_len - max_len = max(self.max_len, cur_len) - if max_len * (len(self.batch) + 1) > self._batch_size: - result = self.batch - self.batch = [info] - self.max_len = cur_len - return result - else: - self.max_len = max_len - self.batch.append(info) - - -class SampleInfo(object): - def __init__(self, i, max_len, min_len): - self.i = i - self.min_len = min_len - self.max_len = max_len - - -class MinMaxFilter(object): - def __init__(self, max_len, min_len, underlying_creator): - self._min_len = min_len - self._max_len = max_len - self._creator = underlying_creator - - def append(self, info): - if info.max_len > self._max_len or info.min_len < self._min_len: - return - else: - return self._creator.append(info) - - @property - def batch(self): - return self._creator.batch - - -class DataReader(object): - """ - The data reader loads all data from files and produces batches of data - in the way corresponding to settings. - - An example of returning a generator producing data batches whose data - is shuffled in each pass and sorted in each pool: - - ``` - train_data = DataReader( - src_vocab_fpath='data/src_vocab_file', - trg_vocab_fpath='data/trg_vocab_file', - fpattern='data/part-*', - use_token_batch=True, - batch_size=2000, - pool_size=10000, - sort_type=SortType.POOL, - shuffle=True, - shuffle_batch=True, - start_mark='', - end_mark='', - unk_mark='', - clip_last_batch=False).batch_generator - ``` - - :param src_vocab_fpath: The path of vocabulary file of source language. - :type src_vocab_fpath: basestring - :param trg_vocab_fpath: The path of vocabulary file of target language. - :type trg_vocab_fpath: basestring - :param fpattern: The pattern to match data files. - :type fpattern: basestring - :param batch_size: The number of sequences contained in a mini-batch. - or the maximum number of tokens (include paddings) contained in a - mini-batch. - :type batch_size: int - :param pool_size: The size of pool buffer. - :type pool_size: int - :param sort_type: The grain to sort by length: 'global' for all - instances; 'pool' for instances in pool; 'none' for no sort. - :type sort_type: basestring - :param clip_last_batch: Whether to clip the last uncompleted batch. - :type clip_last_batch: bool - :param tar_fname: The data file in tar if fpattern matches a tar file. - :type tar_fname: basestring - :param min_length: The minimum length used to filt sequences. - :type min_length: int - :param max_length: The maximum length used to filt sequences. - :type max_length: int - :param shuffle: Whether to shuffle all instances. - :type shuffle: bool - :param shuffle_batch: Whether to shuffle the generated batches. - :type shuffle_batch: bool - :param use_token_batch: Whether to produce batch data according to - token number. - :type use_token_batch: bool - :param field_delimiter: The delimiter used to split source and target in - each line of data file. - :type field_delimiter: basestring - :param token_delimiter: The delimiter used to split tokens in source or - target sentences. - :type token_delimiter: basestring - :param start_mark: The token representing for the beginning of - sentences in dictionary. - :type start_mark: basestring - :param end_mark: The token representing for the end of sentences - in dictionary. - :type end_mark: basestring - :param unk_mark: The token representing for unknown word in dictionary. - :type unk_mark: basestring - :param seed: The seed for random. - :type seed: int - """ - - def __init__(self, - src_vocab_fpath, - trg_vocab_fpath, - fpattern, - batch_size, - pool_size, - sort_type=SortType.GLOBAL, - clip_last_batch=True, - tar_fname=None, - min_length=0, - max_length=100, - shuffle=True, - shuffle_batch=False, - use_token_batch=False, - field_delimiter="\t", - token_delimiter=" ", - start_mark="", - end_mark="", - unk_mark="", - seed=0): - self._src_vocab = self.load_dict(src_vocab_fpath) - self._only_src = True - if trg_vocab_fpath is not None: - self._trg_vocab = self.load_dict(trg_vocab_fpath) - self._only_src = False - self._pool_size = pool_size - self._batch_size = batch_size - self._use_token_batch = use_token_batch - self._sort_type = sort_type - self._clip_last_batch = clip_last_batch - self._shuffle = shuffle - self._shuffle_batch = shuffle_batch - self._min_length = min_length - self._max_length = max_length - self._field_delimiter = field_delimiter - self._token_delimiter = token_delimiter - self.load_src_trg_ids(end_mark, fpattern, start_mark, tar_fname, - unk_mark) - self._random = np.random - self._random.seed(seed) - - def load_src_trg_ids(self, end_mark, fpattern, start_mark, tar_fname, - unk_mark): - converters = [ - Converter( - vocab=self._src_vocab, - beg=self._src_vocab[start_mark], - end=self._src_vocab[end_mark], - unk=self._src_vocab[unk_mark], - delimiter=self._token_delimiter, - add_beg=False) - ] - if not self._only_src: - converters.append( - Converter( - vocab=self._trg_vocab, - beg=self._trg_vocab[start_mark], - end=self._trg_vocab[end_mark], - unk=self._trg_vocab[unk_mark], - delimiter=self._token_delimiter, - add_beg=True)) - - converters = ComposedConverter(converters) - - self._src_seq_ids = [] - self._trg_seq_ids = None if self._only_src else [] - self._sample_infos = [] - - for i, line in enumerate(self._load_lines(fpattern, tar_fname)): - src_trg_ids = converters(line) - self._src_seq_ids.append(src_trg_ids[0]) - lens = [len(src_trg_ids[0])] - if not self._only_src: - self._trg_seq_ids.append(src_trg_ids[1]) - lens.append(len(src_trg_ids[1])) - self._sample_infos.append(SampleInfo(i, max(lens), min(lens))) - - def _load_lines(self, fpattern, tar_fname): - fpaths = glob.glob(fpattern) - - if len(fpaths) == 1 and tarfile.is_tarfile(fpaths[0]): - if tar_fname is None: - raise Exception("If tar file provided, please set tar_fname.") - - f = tarfile.open(fpaths[0], "r") - for line in f.extractfile(tar_fname): - fields = line.strip("\n").split(self._field_delimiter) - if (not self._only_src and len(fields) == 2) or ( - self._only_src and len(fields) == 1): - yield fields - else: - for fpath in fpaths: - if not os.path.isfile(fpath): - raise IOError("Invalid file: %s" % fpath) - - with open(fpath, "rb") as f: - for line in f: - if six.PY3: - line = line.decode("utf8", errors="ignore") - fields = line.strip("\n").split(self._field_delimiter) - if (not self._only_src and len(fields) == 2) or ( - self._only_src and len(fields) == 1): - yield fields - - @staticmethod - def load_dict(dict_path, reverse=False): - word_dict = {} - with open(dict_path, "rb") as fdict: - for idx, line in enumerate(fdict): - if six.PY3: - line = line.decode("utf8", errors="ignore") - if reverse: - word_dict[idx] = line.strip("\n") - else: - word_dict[line.strip("\n")] = idx - return word_dict - - def batch_generator(self): - # global sort or global shuffle - if self._sort_type == SortType.GLOBAL: - infos = sorted(self._sample_infos, key=lambda x: x.max_len) - else: - if self._shuffle: - infos = self._sample_infos - self._random.shuffle(infos) - else: - infos = self._sample_infos - - if self._sort_type == SortType.POOL: - reverse = True - for i in range(0, len(infos), self._pool_size): - # to avoid placing short next to long sentences - reverse = not reverse - infos[i:i + self._pool_size] = sorted( - infos[i:i + self._pool_size], - key=lambda x: x.max_len, - reverse=reverse) - - # concat batch - batches = [] - batch_creator = TokenBatchCreator( - self._batch_size - ) if self._use_token_batch else SentenceBatchCreator(self._batch_size) - batch_creator = MinMaxFilter(self._max_length, self._min_length, - batch_creator) - - for info in infos: - batch = batch_creator.append(info) - if batch is not None: - batches.append(batch) - - if not self._clip_last_batch and len(batch_creator.batch) != 0: - batches.append(batch_creator.batch) - - if self._shuffle_batch: - self._random.shuffle(batches) - - for batch in batches: - batch_ids = [info.i for info in batch] - - if self._only_src: - yield [[self._src_seq_ids[idx]] for idx in batch_ids] - else: - yield [(self._src_seq_ids[idx], self._trg_seq_ids[idx][:-1], - self._trg_seq_ids[idx][1:]) for idx in batch_ids] diff --git a/PaddleNLP/unarchived/neural_machine_translation/transformer/train.py b/PaddleNLP/unarchived/neural_machine_translation/transformer/train.py deleted file mode 100644 index fa719c94f4e64b2936ffcece3aa9b3aabc4fe960..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/neural_machine_translation/transformer/train.py +++ /dev/null @@ -1,776 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import ast -import copy -import logging -import multiprocessing -import os -import six -import sys -import time - -import numpy as np -import paddle.fluid as fluid - -import reader -from config import * -from model import transformer, position_encoding_init - - -def parse_args(): - parser = argparse.ArgumentParser("Training for Transformer.") - parser.add_argument( - "--src_vocab_fpath", - type=str, - required=True, - help="The path of vocabulary file of source language.") - parser.add_argument( - "--trg_vocab_fpath", - type=str, - required=True, - help="The path of vocabulary file of target language.") - parser.add_argument( - "--train_file_pattern", - type=str, - required=True, - help="The pattern to match training data files.") - parser.add_argument( - "--val_file_pattern", - type=str, - help="The pattern to match validation data files.") - parser.add_argument( - "--use_token_batch", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to " - "produce batch data according to token number.") - parser.add_argument( - "--batch_size", - type=int, - default=4096, - help="The number of sequences contained in a mini-batch, or the maximum " - "number of tokens (include paddings) contained in a mini-batch. Note " - "that this represents the number on single device and the actual batch " - "size for multi-devices will multiply the device number.") - parser.add_argument( - "--pool_size", - type=int, - default=200000, - help="The buffer size to pool data.") - parser.add_argument( - "--sort_type", - default="pool", - choices=("global", "pool", "none"), - help="The grain to sort by length: global for all instances; pool for " - "instances in pool; none for no sort.") - parser.add_argument( - "--shuffle", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to shuffle instances in each pass.") - parser.add_argument( - "--shuffle_batch", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to shuffle the data batches.") - parser.add_argument( - "--special_token", - type=str, - default=["", "", ""], - nargs=3, - help="The , and tokens in the dictionary.") - parser.add_argument( - "--token_delimiter", - type=lambda x: str(x.encode().decode("unicode-escape")), - default=" ", - help="The delimiter used to split tokens in source or target sentences. " - "For EN-DE BPE data we provided, use spaces as token delimiter. ") - parser.add_argument( - 'opts', - help='See config.py for all options', - default=None, - nargs=argparse.REMAINDER) - parser.add_argument( - '--local', - type=ast.literal_eval, - default=True, - help='Whether to run as local mode.') - parser.add_argument( - '--device', - type=str, - default='GPU', - choices=['CPU', 'GPU'], - help="The device type.") - parser.add_argument( - '--update_method', - choices=("pserver", "nccl2"), - default="pserver", - help='Update method.') - parser.add_argument( - '--sync', type=ast.literal_eval, default=True, help="sync mode.") - parser.add_argument( - "--enable_ce", - type=ast.literal_eval, - default=False, - help="The flag indicating whether to run the task " - "for continuous evaluation.") - parser.add_argument( - "--use_py_reader", - type=ast.literal_eval, - default=True, - help="The flag indicating whether to use py_reader.") - parser.add_argument( - "--fetch_steps", - type=int, - default=100, - help="The frequency to fetch and print output.") - - args = parser.parse_args() - # Append args related to dict - src_dict = reader.DataReader.load_dict(args.src_vocab_fpath) - trg_dict = reader.DataReader.load_dict(args.trg_vocab_fpath) - dict_args = [ - "src_vocab_size", str(len(src_dict)), "trg_vocab_size", - str(len(trg_dict)), "bos_idx", str(src_dict[args.special_token[0]]), - "eos_idx", str(src_dict[args.special_token[1]]), "unk_idx", - str(src_dict[args.special_token[2]]) - ] - merge_cfg_from_list(args.opts + dict_args, - [TrainTaskConfig, ModelHyperParams]) - return args - - -def append_nccl2_prepare(startup_prog, trainer_id, worker_endpoints, - current_endpoint): - assert (trainer_id >= 0 and len(worker_endpoints) > 1 and - current_endpoint in worker_endpoints) - eps = copy.deepcopy(worker_endpoints) - eps.remove(current_endpoint) - nccl_id_var = startup_prog.global_block().create_var( - name="NCCLID", persistable=True, type=fluid.core.VarDesc.VarType.RAW) - startup_prog.global_block().append_op( - type="gen_nccl_id", - inputs={}, - outputs={"NCCLID": nccl_id_var}, - attrs={ - "endpoint": current_endpoint, - "endpoint_list": eps, - "trainer_id": trainer_id - }) - return nccl_id_var - - -def pad_batch_data(insts, - pad_idx, - n_head, - is_target=False, - is_label=False, - return_attn_bias=True, - return_max_len=True, - return_num_token=False): - """ - Pad the instances to the max sequence length in batch, and generate the - corresponding position data and attention bias. - """ - return_list = [] - max_len = max(len(inst) for inst in insts) - # Any token included in dict can be used to pad, since the paddings' loss - # will be masked out by weights and make no effect on parameter gradients. - inst_data = np.array( - [inst + [pad_idx] * (max_len - len(inst)) for inst in insts]) - return_list += [inst_data.astype("int64").reshape([-1, 1])] - if is_label: # label weight - inst_weight = np.array( - [[1.] * len(inst) + [0.] * (max_len - len(inst)) for inst in insts]) - return_list += [inst_weight.astype("float32").reshape([-1, 1])] - else: # position data - inst_pos = np.array([ - list(range(0, len(inst))) + [0] * (max_len - len(inst)) - for inst in insts - ]) - return_list += [inst_pos.astype("int64").reshape([-1, 1])] - if return_attn_bias: - if is_target: - # This is used to avoid attention on paddings and subsequent - # words. - slf_attn_bias_data = np.ones((inst_data.shape[0], max_len, max_len)) - slf_attn_bias_data = np.triu(slf_attn_bias_data, - 1).reshape([-1, 1, max_len, max_len]) - slf_attn_bias_data = np.tile(slf_attn_bias_data, - [1, n_head, 1, 1]) * [-1e9] - else: - # This is used to avoid attention on paddings. - slf_attn_bias_data = np.array([[0] * len(inst) + [-1e9] * - (max_len - len(inst)) - for inst in insts]) - slf_attn_bias_data = np.tile( - slf_attn_bias_data.reshape([-1, 1, 1, max_len]), - [1, n_head, max_len, 1]) - return_list += [slf_attn_bias_data.astype("float32")] - if return_max_len: - return_list += [max_len] - if return_num_token: - num_token = 0 - for inst in insts: - num_token += len(inst) - return_list += [num_token] - return return_list if len(return_list) > 1 else return_list[0] - - -def prepare_batch_input(insts, data_input_names, src_pad_idx, trg_pad_idx, - n_head, d_model): - """ - Put all padded data needed by training into a dict. - """ - src_word, src_pos, src_slf_attn_bias, src_max_len = pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False) - src_word = src_word.reshape(-1, src_max_len, 1) - src_pos = src_pos.reshape(-1, src_max_len, 1) - trg_word, trg_pos, trg_slf_attn_bias, trg_max_len = pad_batch_data( - [inst[1] for inst in insts], trg_pad_idx, n_head, is_target=True) - trg_word = trg_word.reshape(-1, trg_max_len, 1) - trg_pos = trg_pos.reshape(-1, trg_max_len, 1) - - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, trg_max_len, 1]).astype("float32") - - lbl_word, lbl_weight, num_token = pad_batch_data( - [inst[2] for inst in insts], - trg_pad_idx, - n_head, - is_target=False, - is_label=True, - return_attn_bias=False, - return_max_len=False, - return_num_token=True) - - data_input_dict = dict( - zip(data_input_names, [ - src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, lbl_weight - ])) - - return data_input_dict, np.asarray([num_token], dtype="float32") - - -def prepare_data_generator(args, - is_test, - count, - pyreader, - py_reader_provider_wrapper, - place=None): - """ - Data generator wrapper for DataReader. If use py_reader, set the data - provider for py_reader - """ - data_reader = reader.DataReader( - fpattern=args.val_file_pattern if is_test else args.train_file_pattern, - src_vocab_fpath=args.src_vocab_fpath, - trg_vocab_fpath=args.trg_vocab_fpath, - token_delimiter=args.token_delimiter, - use_token_batch=args.use_token_batch, - batch_size=args.batch_size * (1 if args.use_token_batch else count), - pool_size=args.pool_size, - sort_type=args.sort_type, - shuffle=args.shuffle, - shuffle_batch=args.shuffle_batch, - start_mark=args.special_token[0], - end_mark=args.special_token[1], - unk_mark=args.special_token[2], - # count start and end tokens out - max_length=ModelHyperParams.max_length - 2, - clip_last_batch=False).batch_generator - - def stack(data_reader, count, clip_last=True): - def __impl__(): - res = [] - for item in data_reader(): - res.append(item) - if len(res) == count: - yield res - res = [] - if len(res) == count: - yield res - elif not clip_last: - data = [] - for item in res: - data += item - if len(data) > count: - inst_num_per_part = len(data) // count - yield [ - data[inst_num_per_part * i:inst_num_per_part * (i + 1)] - for i in range(count) - ] - - return __impl__ - - def split(data_reader, count): - def __impl__(): - for item in data_reader(): - inst_num_per_part = len(item) // count - for i in range(count): - yield item[inst_num_per_part * i:inst_num_per_part * (i + 1 - )] - - return __impl__ - - if not args.use_token_batch: - # to make data on each device have similar token number - data_reader = split(data_reader, count) - if args.use_py_reader: - pyreader.decorate_tensor_provider( - py_reader_provider_wrapper(data_reader, place)) - data_reader = None - else: # Data generator for multi-devices - data_reader = stack(data_reader, count) - return data_reader - - -def prepare_feed_dict_list(data_generator, init_flag, count): - """ - Prepare the list of feed dict for multi-devices. - """ - feed_dict_list = [] - if data_generator is not None: # use_py_reader == False - data_input_names = encoder_data_input_fields + \ - decoder_data_input_fields[:-1] + label_data_input_fields - data = next(data_generator) - for idx, data_buffer in enumerate(data): - data_input_dict, num_token = prepare_batch_input( - data_buffer, data_input_names, ModelHyperParams.eos_idx, - ModelHyperParams.eos_idx, ModelHyperParams.n_head, - ModelHyperParams.d_model) - feed_dict_list.append(data_input_dict) - if init_flag: - for idx in range(count): - pos_enc_tables = dict() - for pos_enc_param_name in pos_enc_param_names: - pos_enc_tables[pos_enc_param_name] = position_encoding_init( - ModelHyperParams.max_length + 1, ModelHyperParams.d_model) - if len(feed_dict_list) <= idx: - feed_dict_list.append(pos_enc_tables) - else: - feed_dict_list[idx] = dict( - list(pos_enc_tables.items()) + list(feed_dict_list[idx] - .items())) - - return feed_dict_list if len(feed_dict_list) == count else None - - -def py_reader_provider_wrapper(data_reader, place): - """ - Data provider needed by fluid.layers.py_reader. - """ - - def py_reader_provider(): - data_input_names = encoder_data_input_fields + \ - decoder_data_input_fields[:-1] + label_data_input_fields - for batch_id, data in enumerate(data_reader()): - data_input_dict, num_token = prepare_batch_input( - data, data_input_names, ModelHyperParams.eos_idx, - ModelHyperParams.eos_idx, ModelHyperParams.n_head, - ModelHyperParams.d_model) - yield [data_input_dict[item] for item in data_input_names] - - return py_reader_provider - - -def test_context(exe, train_exe, dev_count): - # Context to do validation. - test_prog = fluid.Program() - startup_prog = fluid.Program() - if args.enable_ce: - test_prog.random_seed = 1000 - startup_prog.random_seed = 1000 - with fluid.program_guard(test_prog, startup_prog): - with fluid.unique_name.guard(): - sum_cost, avg_cost, predict, token_num, pyreader = transformer( - ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - TrainTaskConfig.label_smooth_eps, - use_py_reader=args.use_py_reader, - is_test=True) - test_prog = test_prog.clone(for_test=True) - test_data = prepare_data_generator( - args, - is_test=True, - count=dev_count, - pyreader=pyreader, - py_reader_provider_wrapper=py_reader_provider_wrapper) - - exe.run(startup_prog) # to init pyreader for testing - if TrainTaskConfig.ckpt_path: - fluid.io.load_persistables( - exe, TrainTaskConfig.ckpt_path, main_program=test_prog) - - exec_strategy = fluid.ExecutionStrategy() - exec_strategy.use_experimental_executor = True - build_strategy = fluid.BuildStrategy() - test_exe = fluid.ParallelExecutor( - use_cuda=TrainTaskConfig.use_gpu, - main_program=test_prog, - build_strategy=build_strategy, - exec_strategy=exec_strategy, - share_vars_from=train_exe) - - def test(exe=test_exe, pyreader=pyreader): - test_total_cost = 0 - test_total_token = 0 - - if args.use_py_reader: - pyreader.start() - data_generator = None - else: - data_generator = test_data() - while True: - try: - feed_dict_list = prepare_feed_dict_list(data_generator, False, - dev_count) - outs = test_exe.run(fetch_list=[sum_cost.name, token_num.name], - feed=feed_dict_list) - except (StopIteration, fluid.core.EOFException): - # The current pass is over. - if args.use_py_reader: - pyreader.reset() - break - sum_cost_val, token_num_val = np.array(outs[0]), np.array(outs[1]) - test_total_cost += sum_cost_val.sum() - test_total_token += token_num_val.sum() - test_avg_cost = test_total_cost / test_total_token - test_ppl = np.exp([min(test_avg_cost, 100)]) - return test_avg_cost, test_ppl - - return test - - -def train_loop(exe, - train_prog, - startup_prog, - dev_count, - sum_cost, - avg_cost, - token_num, - predict, - pyreader, - nccl2_num_trainers=1, - nccl2_trainer_id=0): - # Initialize the parameters. - if TrainTaskConfig.ckpt_path: - exe.run(startup_prog) # to init pyreader for training - logging.info("load checkpoint from {}".format( - TrainTaskConfig.ckpt_path)) - fluid.io.load_persistables( - exe, TrainTaskConfig.ckpt_path, main_program=train_prog) - else: - logging.info("init fluid.framework.default_startup_program") - exe.run(startup_prog) - - logging.info("begin reader") - train_data = prepare_data_generator( - args, - is_test=False, - count=dev_count, - pyreader=pyreader, - py_reader_provider_wrapper=py_reader_provider_wrapper) - - # For faster executor - exec_strategy = fluid.ExecutionStrategy() - exec_strategy.use_experimental_executor = True - exec_strategy.num_iteration_per_drop_scope = int(args.fetch_steps) - build_strategy = fluid.BuildStrategy() - # Since the token number differs among devices, customize gradient scale to - # use token average cost among multi-devices. and the gradient scale is - # `1 / token_number` for average cost. - # build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized - - logging.info("begin executor") - train_exe = fluid.ParallelExecutor( - use_cuda=TrainTaskConfig.use_gpu, - loss_name=avg_cost.name, - main_program=train_prog, - build_strategy=build_strategy, - exec_strategy=exec_strategy, - num_trainers=nccl2_num_trainers, - trainer_id=nccl2_trainer_id) - - if args.val_file_pattern is not None: - test = test_context(exe, train_exe, dev_count) - - # the best cross-entropy value with label smoothing - loss_normalizer = -((1. - TrainTaskConfig.label_smooth_eps) * np.log( - (1. - TrainTaskConfig.label_smooth_eps - )) + TrainTaskConfig.label_smooth_eps * - np.log(TrainTaskConfig.label_smooth_eps / ( - ModelHyperParams.trg_vocab_size - 1) + 1e-20)) - - step_idx = 0 - init_flag = True - - logging.info("begin train") - for pass_id in six.moves.xrange(TrainTaskConfig.pass_num): - pass_start_time = time.time() - - if args.use_py_reader: - pyreader.start() - data_generator = None - else: - data_generator = train_data() - - batch_id = 0 - while True: - try: - feed_dict_list = prepare_feed_dict_list(data_generator, - init_flag, dev_count) - outs = train_exe.run( - fetch_list=[sum_cost.name, token_num.name] - if step_idx % args.fetch_steps == 0 else [], - feed=feed_dict_list) - - if step_idx % args.fetch_steps == 0: - sum_cost_val, token_num_val = np.array(outs[0]), np.array( - outs[1]) - # sum the cost from multi-devices - total_sum_cost = sum_cost_val.sum() - total_token_num = token_num_val.sum() - total_avg_cost = total_sum_cost / total_token_num - - if step_idx == 0: - logging.info( - "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " - "normalized loss: %f, ppl: %f" % - (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100)]))) - avg_batch_time = time.time() - else: - logging.info( - "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " - "normalized loss: %f, ppl: %f, speed: %.2f step/s" % - (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100)]), - args.fetch_steps / (time.time() - avg_batch_time))) - avg_batch_time = time.time() - - if step_idx % TrainTaskConfig.save_freq == 0 and step_idx > 0: - fluid.io.save_persistables( - exe, - os.path.join(TrainTaskConfig.ckpt_dir, - "latest.checkpoint"), train_prog) - fluid.io.save_params( - exe, - os.path.join(TrainTaskConfig.model_dir, - "iter_" + str(step_idx) + ".infer.model"), - train_prog) - - init_flag = False - batch_id += 1 - step_idx += 1 - except (StopIteration, fluid.core.EOFException): - # The current pass is over. - if args.use_py_reader: - pyreader.reset() - break - - time_consumed = time.time() - pass_start_time - # Validate and save the persistable. - if args.val_file_pattern is not None: - val_avg_cost, val_ppl = test() - logging.info( - "epoch: %d, val avg loss: %f, val normalized loss: %f, val ppl: %f," - " consumed %fs" % (pass_id, val_avg_cost, - val_avg_cost - loss_normalizer, val_ppl, - time_consumed)) - else: - logging.info("epoch: %d, consumed %fs" % (pass_id, time_consumed)) - if not args.enable_ce: - fluid.io.save_persistables( - exe, - os.path.join(TrainTaskConfig.ckpt_dir, - "pass_" + str(pass_id) + ".checkpoint"), - train_prog) - - if args.enable_ce: # For CE - print("kpis\ttrain_cost_card%d\t%f" % (dev_count, total_avg_cost)) - if args.val_file_pattern is not None: - print("kpis\ttest_cost_card%d\t%f" % (dev_count, val_avg_cost)) - print("kpis\ttrain_duration_card%d\t%f" % (dev_count, time_consumed)) - - -def train(args): - # priority: ENV > args > config - is_local = os.getenv("PADDLE_IS_LOCAL", "1") - if is_local == '0': - args.local = False - logging.info(args) - - if args.device == 'CPU': - TrainTaskConfig.use_gpu = False - - training_role = os.getenv("TRAINING_ROLE", "TRAINER") - - if training_role == "PSERVER" or (not TrainTaskConfig.use_gpu): - place = fluid.CPUPlace() - dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - else: - place = fluid.CUDAPlace(0) - dev_count = fluid.core.get_cuda_device_count() - - exe = fluid.Executor(place) - - train_prog = fluid.Program() - startup_prog = fluid.Program() - - if args.enable_ce: - train_prog.random_seed = 1000 - startup_prog.random_seed = 1000 - - with fluid.program_guard(train_prog, startup_prog): - with fluid.unique_name.guard(): - sum_cost, avg_cost, predict, token_num, pyreader = transformer( - ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - TrainTaskConfig.label_smooth_eps, - use_py_reader=args.use_py_reader, - is_test=False) - - optimizer = None - if args.sync: - lr_decay = fluid.layers.learning_rate_scheduler.noam_decay( - ModelHyperParams.d_model, TrainTaskConfig.warmup_steps) - logging.info("before adam") - - with fluid.default_main_program()._lr_schedule_guard(): - learning_rate = lr_decay * TrainTaskConfig.learning_rate - - optimizer = fluid.optimizer.Adam( - learning_rate=learning_rate, - beta1=TrainTaskConfig.beta1, - beta2=TrainTaskConfig.beta2, - epsilon=TrainTaskConfig.eps) - else: - optimizer = fluid.optimizer.SGD(0.003) - optimizer.minimize(avg_cost) - - if args.local: - logging.info("local start_up:") - train_loop(exe, train_prog, startup_prog, dev_count, sum_cost, avg_cost, - token_num, predict, pyreader) - else: - if args.update_method == "nccl2": - trainer_id = int(os.getenv("PADDLE_TRAINER_ID", "0")) - port = os.getenv("PADDLE_PORT") - worker_ips = os.getenv("PADDLE_TRAINERS") - worker_endpoints = [] - for ip in worker_ips.split(","): - worker_endpoints.append(':'.join([ip, port])) - trainers_num = len(worker_endpoints) - current_endpoint = os.getenv("POD_IP") + ":" + port - if trainer_id == 0: - logging.info("train_id == 0, sleep 60s") - time.sleep(60) - logging.info("trainers_num:{}".format(trainers_num)) - logging.info("worker_endpoints:{}".format(worker_endpoints)) - logging.info("current_endpoint:{}".format(current_endpoint)) - append_nccl2_prepare(startup_prog, trainer_id, worker_endpoints, - current_endpoint) - train_loop(exe, train_prog, startup_prog, dev_count, sum_cost, - avg_cost, token_num, predict, pyreader, trainers_num, - trainer_id) - return - - port = os.getenv("PADDLE_PORT", "6174") - pserver_ips = os.getenv("PADDLE_PSERVERS") # ip,ip... - eplist = [] - for ip in pserver_ips.split(","): - eplist.append(':'.join([ip, port])) - pserver_endpoints = ",".join(eplist) # ip:port,ip:port... - trainers = int(os.getenv("PADDLE_TRAINERS_NUM", "0")) - current_endpoint = os.getenv("POD_IP") + ":" + port - trainer_id = int(os.getenv("PADDLE_TRAINER_ID")) - - logging.info("pserver_endpoints:{}".format(pserver_endpoints)) - logging.info("current_endpoint:{}".format(current_endpoint)) - logging.info("trainer_id:{}".format(trainer_id)) - logging.info("pserver_ips:{}".format(pserver_ips)) - logging.info("port:{}".format(port)) - - t = fluid.DistributeTranspiler() - t.transpile( - trainer_id, - pservers=pserver_endpoints, - trainers=trainers, - program=train_prog, - startup_program=startup_prog) - - if training_role == "PSERVER": - logging.info("distributed: pserver started") - current_endpoint = os.getenv("POD_IP") + ":" + os.getenv( - "PADDLE_PORT") - if not current_endpoint: - logging.critical("need env SERVER_ENDPOINT") - exit(1) - pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) - - exe.run(pserver_startup) - exe.run(pserver_prog) - elif training_role == "TRAINER": - logging.info("distributed: trainer started") - trainer_prog = t.get_trainer_program() - - train_loop(exe, train_prog, startup_prog, dev_count, sum_cost, - avg_cost, token_num, predict, pyreader) - else: - logging.critical( - "environment var TRAINER_ROLE should be TRAINER os PSERVER") - exit(1) - - -if __name__ == "__main__": - LOG_FORMAT = "[%(asctime)s %(levelname)s %(filename)s:%(lineno)d] %(message)s" - logging.basicConfig( - stream=sys.stdout, level=logging.DEBUG, format=LOG_FORMAT) - logging.getLogger().setLevel(logging.INFO) - - args = parse_args() - train(args) diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/.run_ce.sh b/PaddleNLP/unarchived/sequence_tagging_for_ner/.run_ce.sh deleted file mode 100755 index e4d1c58bcc369e3164cb16b9f715df2775912762..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/.run_ce.sh +++ /dev/null @@ -1,6 +0,0 @@ -###!/bin/bash -####This file is only used for continuous evaluation. - -export CE_MODE_X=1 -sh data/download.sh -python train.py | python _ce.py diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/README.md b/PaddleNLP/unarchived/sequence_tagging_for_ner/README.md deleted file mode 100644 index 6d4efa9eb19dd708a87d4883dccef5ecb5e11666..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# 命名实体识别 - -以下是本例的简要目录结构及说明： - -```text -. -├── data # 存储运行本例所依赖的数据，从外部获取 -├── network_conf.py # 模型定义 -├── reader.py # 数据读取接口, 从外部获取 -├── README.md # 文档 -├── train.py # 训练脚本 -├── infer.py # 预测脚本 -├── utils.py # 定义通用的函数, 从外部获取 -└── utils_extend.py # 对utils.py的拓展 -``` - - -## 简介，模型详解 - -在PaddlePaddle v2版本[命名实体识别](https://github.com/PaddlePaddle/models/blob/develop/legacy/sequence_tagging_for_ner/README.md)中对于命名实体识别任务有较详细的介绍，在本例中不再重复介绍。 -在模型上，我们沿用了v2版本的模型结构，唯一区别是我们使用LSTM代替原始的RNN。 - -## 数据获取 - -完整数据的获取请参考PaddlePaddle v2版本[命名实体识别](https://github.com/PaddlePaddle/models/blob/develop/legacy/sequence_tagging_for_ner/README.md) 一节中的方式。本例的示例数据同样可以通过运行data/download.sh来获取。 - -## 训练 - -1. 运行 `sh data/download.sh` -2. 修改 `train.py` 的 `main` 函数，指定数据路径 - - ```python - main( - train_data_file="data/train", - test_data_file="data/test", - vocab_file="data/vocab.txt", - target_file="data/target.txt", - emb_file="data/wordVectors.txt", - model_save_dir="models", - num_passes=1000, - use_gpu=False, - parallel=False) - ``` - -3. 运行命令 `python train.py` ，**需要注意：直接运行使用的是示例数据，请替换真实的标记数据。** - - ```text - Pass 127, Batch 9525, Cost 4.0867705, Precision 0.3954984, Recall 0.37846154, F1_score0.38679245 - Pass 127, Batch 9530, Cost 3.137265, Precision 0.42971888, Recall 0.38351256, F1_score0.405303 - Pass 127, Batch 9535, Cost 3.6240938, Precision 0.4272152, Recall 0.41795665, F1_score0.4225352 - Pass 127, Batch 9540, Cost 3.5352352, Precision 0.48464164, Recall 0.4536741, F1_score0.46864685 - Pass 127, Batch 9545, Cost 4.1130385, Precision 0.40131578, Recall 0.3836478, F1_score0.39228293 - Pass 127, Batch 9550, Cost 3.6826708, Precision 0.43333334, Recall 0.43730888, F1_score0.43531203 - Pass 127, Batch 9555, Cost 3.6363933, Precision 0.42424244, Recall 0.3962264, F1_score0.4097561 - Pass 127, Batch 9560, Cost 3.6101768, Precision 0.51363635, Recall 0.353125, F1_score0.41851854 - Pass 127, Batch 9565, Cost 3.5935276, Precision 0.5152439, Recall 0.5, F1_score0.5075075 - Pass 127, Batch 9570, Cost 3.4987144, Precision 0.5, Recall 0.4330218, F1_score0.46410686 - Pass 127, Batch 9575, Cost 3.4659843, Precision 0.39864865, Recall 0.38064516, F1_score0.38943896 - Pass 127, Batch 9580, Cost 3.1702557, Precision 0.5, Recall 0.4490446, F1_score0.47315437 - Pass 127, Batch 9585, Cost 3.1587276, Precision 0.49377593, Recall 0.4089347, F1_score0.4473684 - Pass 127, Batch 9590, Cost 3.5043538, Precision 0.4556962, Recall 0.4600639, F1_score0.45786962 - Pass 127, Batch 9595, Cost 2.981989, Precision 0.44981414, Recall 0.45149255, F1_score0.4506518 - [TrainSet] pass_id:127 pass_precision:[0.46023396] pass_recall:[0.43197003] pass_f1_score:[0.44565433] - [TestSet] pass_id:127 pass_precision:[0.4708409] pass_recall:[0.47971722] pass_f1_score:[0.4752376] - ``` -## 预测 -1. 修改 [infer.py](./infer.py) 的 `infer` 函数，指定：需要测试的模型的路径、测试数据、字典文件，预测标记文件的路径，默认参数如下： - - ```python - infer( - model_path="models/params_pass_0", - batch_size=6, - test_data_file="data/test", - vocab_file="data/vocab.txt", - target_file="data/target.txt", - use_gpu=False - ) - ``` - -2. 在终端运行 `python infer.py`，开始测试，会看到如下预测结果（以下为训练70个pass所得模型的部分预测结果）： - - ```text - leicestershire B-ORG B-LOC - extended O O - their O O - first O O - innings O O - by O O - DGDG O O - runs O O - before O O - being O O - bowled O O - out O O - for O O - 296 O O - with O O - england B-LOC B-LOC - discard O O - andy B-PER B-PER - caddick I-PER I-PER - taking O O - three O O - for O O - DGDG O O - . O O - ``` - - 输出分为三列，以“\t” 分隔，第一列是输入的词语，第二列是标准结果，第三列为生成的标记结果。多条输入序列之间以空行分隔。 - -## 结果示例 - -

-
-图1. 学习曲线, 横轴表示训练轮数，纵轴表示F1值 -

diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/_ce.py b/PaddleNLP/unarchived/sequence_tagging_for_ner/_ce.py deleted file mode 100644 index 0d38fdbbaa52272c777410600a4ff88de9741326..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/_ce.py +++ /dev/null @@ -1,48 +0,0 @@ -####this file is only used for continuous evaluation test! - -import os -import sys -sys.path.append(os.environ['ceroot']) -from kpi import CostKpi, DurationKpi, AccKpi - -#### NOTE kpi.py should shared in models in some way!!!! - -train_acc_kpi = AccKpi('train_precision', 0.005, 0, actived=True) -test_acc_kpi = CostKpi('test_precision', 0.005, 0, actived=True) -train_duration_kpi = DurationKpi('train_duration', 0.05, 0, actived=True) - -tracking_kpis = [ - train_acc_kpi, - test_acc_kpi, - train_duration_kpi, -] - - -def parse_log(log): - for line in log.split('\n'): - fs = line.strip().split('\t') - print(fs) - if len(fs) == 3 and fs[0] == 'kpis': - print("-----%s" % fs) - kpi_name = fs[1] - kpi_value = float(fs[2]) - yield kpi_name, kpi_value - - -def log_to_ce(log): - kpi_tracker = {} - for kpi in tracking_kpis: - kpi_tracker[kpi.name] = kpi - - for (kpi_name, kpi_value) in parse_log(log): - print(kpi_name, kpi_value) - kpi_tracker[kpi_name].add_record(kpi_value) - kpi_tracker[kpi_name].persist() - - -if __name__ == '__main__': - log = sys.stdin.read() - print("*****") - print(log) - print("****") - log_to_ce(log) diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/data/download.sh b/PaddleNLP/unarchived/sequence_tagging_for_ner/data/download.sh deleted file mode 100755 index f83eb894f4c250577db9919b8865db15593b7824..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/data/download.sh +++ /dev/null @@ -1,17 +0,0 @@ -if [ -f assignment2.zip ]; then - echo "data exist" - exit 0 -else - wget http://cs224d.stanford.edu/assignment2/assignment2.zip -fi - -if [ $? -eq 0 ];then - unzip assignment2.zip - cp assignment2_release/data/ner/wordVectors.txt . - cp assignment2_release/data/ner/vocab.txt . - rm -rf assignment2_release -else - echo "download data error!" >> /dev/stderr - exit 1 -fi - diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/data/target.txt b/PaddleNLP/unarchived/sequence_tagging_for_ner/data/target.txt deleted file mode 100644 index e0fa4d8f6654be07b4d1188750abb861d7c6f264..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/data/target.txt +++ /dev/null @@ -1,9 +0,0 @@ -B-LOC -I-LOC -B-MISC -I-MISC -B-ORG -I-ORG -B-PER -I-PER -O diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/data/test b/PaddleNLP/unarchived/sequence_tagging_for_ner/data/test deleted file mode 100644 index 66163e1a869d57303117dd94d59ff01be05de8f7..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/data/test +++ /dev/null @@ -1,128 +0,0 @@ -CRICKET NNP I-NP O -- : O O -LEICESTERSHIRE NNP I-NP I-ORG -TAKE NNP I-NP O -OVER IN I-PP O -AT NNP I-NP O -TOP NNP I-NP O -AFTER NNP I-NP O -INNINGS NNP I-NP O -VICTORY NN I-NP O -. . O O - -LONDON NNP I-NP I-LOC -1996-08-30 CD I-NP O - -West NNP I-NP I-MISC -Indian NNP I-NP I-MISC -all-rounder NN I-NP O -Phil NNP I-NP I-PER -Simmons NNP I-NP I-PER -took VBD I-VP O -four CD I-NP O -for IN I-PP O -38 CD I-NP O -on IN I-PP O -Friday NNP I-NP O -as IN I-PP O -Leicestershire NNP I-NP I-ORG -beat VBD I-VP O -Somerset NNP I-NP I-ORG -by IN I-PP O -an DT I-NP O -innings NN I-NP O -and CC O O -39 CD I-NP O -runs NNS I-NP O -in IN I-PP O -two CD I-NP O -days NNS I-NP O -to TO I-VP O -take VB I-VP O -over IN I-PP O -at IN B-PP O -the DT I-NP O -head NN I-NP O -of IN I-PP O -the DT I-NP O -county NN I-NP O -championship NN I-NP O -. . O O - -Their PRP$ I-NP O -stay NN I-NP O -on IN I-PP O -top NN I-NP O -, , O O -though RB I-ADVP O -, , O O -may MD I-VP O -be VB I-VP O -short-lived JJ I-ADJP O -as IN I-PP O -title NN I-NP O -rivals NNS I-NP O -Essex NNP I-NP I-ORG -, , O O -Derbyshire NNP I-NP I-ORG -and CC I-NP O -Surrey NNP I-NP I-ORG -all DT O O -closed VBD I-VP O -in RP I-PRT O -on IN I-PP O -victory NN I-NP O -while IN I-SBAR O -Kent NNP I-NP I-ORG -made VBD I-VP O -up RP I-PRT O -for IN I-PP O -lost VBN I-NP O -time NN I-NP O -in IN I-PP O -their PRP$ I-NP O -rain-affected JJ I-NP O -match NN I-NP O -against IN I-PP O -Nottinghamshire NNP I-NP I-ORG -. . O O - -After IN I-PP O -bowling VBG I-NP O -Somerset NNP I-NP I-ORG -out RP I-PRT O -for IN I-PP O -83 CD I-NP O -on IN I-PP O -the DT I-NP O -opening NN I-NP O -morning NN I-NP O -at IN I-PP O -Grace NNP I-NP I-LOC -Road NNP I-NP I-LOC -, , O O -Leicestershire NNP I-NP I-ORG -extended VBD I-VP O -their PRP$ I-NP O -first JJ I-NP O -innings NN I-NP O -by IN I-PP O -94 CD I-NP O -runs VBZ I-VP O -before IN I-PP O -being VBG I-VP O -bowled VBD I-VP O -out RP I-PRT O -for IN I-PP O -296 CD I-NP O -with IN I-PP O -England NNP I-NP I-LOC -discard VBP I-VP O -Andy NNP I-NP I-PER -Caddick NNP I-NP I-PER -taking VBG I-VP O -three CD I-NP O -for IN I-PP O -83 CD I-NP O -. . O O - diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/data/train b/PaddleNLP/unarchived/sequence_tagging_for_ner/data/train deleted file mode 100644 index cbf3e678c555a3b6db26fd14e38889f040f048ca..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/data/train +++ /dev/null @@ -1,139 +0,0 @@ -EU NNP I-NP I-ORG -rejects VBZ I-VP O -German JJ I-NP I-MISC -call NN I-NP O -to TO I-VP O -boycott VB I-VP O -British JJ I-NP I-MISC -lamb NN I-NP O -. . O O - -Peter NNP I-NP I-PER -Blackburn NNP I-NP I-PER - -BRUSSELS NNP I-NP I-LOC -1996-08-22 CD I-NP O - -The DT I-NP O -European NNP I-NP I-ORG -Commission NNP I-NP I-ORG -said VBD I-VP O -on IN I-PP O -Thursday NNP I-NP O -it PRP B-NP O -disagreed VBD I-VP O -with IN I-PP O -German JJ I-NP I-MISC -advice NN I-NP O -to TO I-PP O -consumers NNS I-NP O -to TO I-VP O -shun VB I-VP O -British JJ I-NP I-MISC -lamb NN I-NP O -until IN I-SBAR O -scientists NNS I-NP O -determine VBP I-VP O -whether IN I-SBAR O -mad JJ I-NP O -cow NN I-NP O -disease NN I-NP O -can MD I-VP O -be VB I-VP O -transmitted VBN I-VP O -to TO I-PP O -sheep NN I-NP O -. . O O - -Germany NNP I-NP I-LOC -'s POS B-NP O -representative NN I-NP O -to TO I-PP O -the DT I-NP O -European NNP I-NP I-ORG -Union NNP I-NP I-ORG -'s POS B-NP O -veterinary JJ I-NP O -committee NN I-NP O -Werner NNP I-NP I-PER -Zwingmann NNP I-NP I-PER -said VBD I-VP O -on IN I-PP O -Wednesday NNP I-NP O -consumers NNS I-NP O -should MD I-VP O -buy VB I-VP O -sheepmeat NN I-NP O -from IN I-PP O -countries NNS I-NP O -other JJ I-ADJP O -than IN I-PP O -Britain NNP I-NP I-LOC -until IN I-SBAR O -the DT I-NP O -scientific JJ I-NP O -advice NN I-NP O -was VBD I-VP O -clearer JJR I-ADJP O -. . O O - -" " O O -We PRP I-NP O -do VBP I-VP O -n't RB I-VP O -support VB I-VP O -any DT I-NP O -such JJ I-NP O -recommendation NN I-NP O -because IN I-SBAR O -we PRP I-NP O -do VBP I-VP O -n't RB I-VP O -see VB I-VP O -any DT I-NP O -grounds NNS I-NP O -for IN I-PP O -it PRP I-NP O -, , O O -" " O O -the DT I-NP O -Commission NNP I-NP I-ORG -'s POS B-NP O -chief JJ I-NP O -spokesman NN I-NP O -Nikolaus NNP I-NP I-PER -van NNP I-NP I-PER -der FW I-NP I-PER -Pas NNP I-NP I-PER -told VBD I-VP O -a DT I-NP O -news NN I-NP O -briefing NN I-NP O -. . O O - -He PRP I-NP O -said VBD I-VP O -further JJ I-NP O -scientific JJ I-NP O -study NN I-NP O -was VBD I-VP O -required VBN I-VP O -and CC O O -if IN I-SBAR O -it PRP I-NP O -was VBD I-VP O -found VBN I-VP O -that IN I-SBAR O -action NN I-NP O -was VBD I-VP O -needed VBN I-VP O -it PRP I-NP O -should MD I-VP O -be VB I-VP O -taken VBN I-VP O -by IN I-PP O -the DT I-NP O -European NNP I-NP I-ORG -Union NNP I-NP I-ORG -. . O O - diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/imgs/convergence_curve.png b/PaddleNLP/unarchived/sequence_tagging_for_ner/imgs/convergence_curve.png deleted file mode 100644 index 6b862b751dd7ec0ef761dce78b9515769366d5f4..0000000000000000000000000000000000000000 Binary files a/PaddleNLP/unarchived/sequence_tagging_for_ner/imgs/convergence_curve.png and /dev/null differ diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/infer.py b/PaddleNLP/unarchived/sequence_tagging_for_ner/infer.py deleted file mode 100644 index 44f547f0ca53fc91648364ba80965db8e30c2cb8..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/infer.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import print_function - -import numpy as np -import six - -import paddle -import paddle.fluid as fluid - -from network_conf import ner_net -import reader -from utils import load_dict, load_reverse_dict -from utils_extend import to_lodtensor - - -def infer(model_path, batch_size, test_data_file, vocab_file, target_file, - use_gpu): - """ - use the model under model_path to predict the test data, the result will be printed on the screen - - return nothing - """ - word_dict = load_dict(vocab_file) - word_reverse_dict = load_reverse_dict(vocab_file) - - label_dict = load_dict(target_file) - label_reverse_dict = load_reverse_dict(target_file) - - test_data = paddle.batch( - reader.data_reader(test_data_file, word_dict, label_dict), - batch_size=batch_size) - place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - - inference_scope = fluid.Scope() - with fluid.scope_guard(inference_scope): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(model_path, exe) - for data in test_data(): - word = to_lodtensor([x[0] for x in data], place) - mark = to_lodtensor([x[1] for x in data], place) - crf_decode = exe.run(inference_program, - feed={"word": word, - "mark": mark}, - fetch_list=fetch_targets, - return_numpy=False) - lod_info = (crf_decode[0].lod())[0] - np_data = np.array(crf_decode[0]) - assert len(data) == len(lod_info) - 1 - for sen_index in six.moves.xrange(len(data)): - assert len(data[sen_index][0]) == lod_info[ - sen_index + 1] - lod_info[sen_index] - word_index = 0 - for tag_index in six.moves.xrange(lod_info[sen_index], - lod_info[sen_index + 1]): - word = word_reverse_dict[data[sen_index][0][word_index]] - gold_tag = label_reverse_dict[data[sen_index][2][ - word_index]] - tag = label_reverse_dict[np_data[tag_index][0]] - print(word + "\t" + gold_tag + "\t" + tag) - word_index += 1 - print("") - - -if __name__ == "__main__": - infer( - model_path="models/params_pass_0", - batch_size=6, - test_data_file="data/test", - vocab_file="data/vocab.txt", - target_file="data/target.txt", - use_gpu=False) diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/network_conf.py b/PaddleNLP/unarchived/sequence_tagging_for_ner/network_conf.py deleted file mode 100644 index 2747176964f2ce920db3a2b2dd553b130546f8a5..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/network_conf.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import math - -import paddle.fluid as fluid -from paddle.fluid.initializer import NormalInitializer - -from utils import logger, load_dict, get_embedding - - -def ner_net(word_dict_len, label_dict_len, parallel, stack_num=2): - mark_dict_len = 2 - word_dim = 50 - mark_dim = 5 - hidden_dim = 300 - IS_SPARSE = True - embedding_name = 'emb' - - def _net_conf(word, mark, target): - word_embedding = fluid.layers.embedding( - input=word, - size=[word_dict_len, word_dim], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr=fluid.ParamAttr( - name=embedding_name, trainable=False)) - - mark_embedding = fluid.layers.embedding( - input=mark, - size=[mark_dict_len, mark_dim], - dtype='float32', - is_sparse=IS_SPARSE) - - word_caps_vector = fluid.layers.concat( - input=[word_embedding, mark_embedding], axis=1) - mix_hidden_lr = 1 - - rnn_para_attr = fluid.ParamAttr( - initializer=NormalInitializer( - loc=0.0, scale=0.0), - learning_rate=mix_hidden_lr) - hidden_para_attr = fluid.ParamAttr( - initializer=NormalInitializer( - loc=0.0, scale=(1. / math.sqrt(hidden_dim) / 3)), - learning_rate=mix_hidden_lr) - - hidden = fluid.layers.fc( - input=word_caps_vector, - name="__hidden00__", - size=hidden_dim, - act="tanh", - bias_attr=fluid.ParamAttr(initializer=NormalInitializer( - loc=0.0, scale=(1. / math.sqrt(hidden_dim) / 3))), - param_attr=fluid.ParamAttr(initializer=NormalInitializer( - loc=0.0, scale=(1. / math.sqrt(hidden_dim) / 3)))) - fea = [] - for direction in ["fwd", "bwd"]: - for i in range(stack_num): - if i != 0: - hidden = fluid.layers.fc( - name="__hidden%02d_%s__" % (i, direction), - size=hidden_dim, - act="stanh", - bias_attr=fluid.ParamAttr(initializer=NormalInitializer( - loc=0.0, scale=1.0)), - input=[hidden, rnn[0], rnn[1]], - param_attr=[ - hidden_para_attr, rnn_para_attr, rnn_para_attr - ]) - rnn = fluid.layers.dynamic_lstm( - name="__rnn%02d_%s__" % (i, direction), - input=hidden, - size=hidden_dim, - candidate_activation='relu', - gate_activation='sigmoid', - cell_activation='sigmoid', - bias_attr=fluid.ParamAttr(initializer=NormalInitializer( - loc=0.0, scale=1.0)), - is_reverse=(i % 2) if direction == "fwd" else not i % 2, - param_attr=rnn_para_attr) - fea += [hidden, rnn[0], rnn[1]] - - rnn_fea = fluid.layers.fc( - size=hidden_dim, - bias_attr=fluid.ParamAttr(initializer=NormalInitializer( - loc=0.0, scale=(1. / math.sqrt(hidden_dim) / 3))), - act="stanh", - input=fea, - param_attr=[hidden_para_attr, rnn_para_attr, rnn_para_attr] * 2) - - emission = fluid.layers.fc( - size=label_dict_len, - input=rnn_fea, - param_attr=fluid.ParamAttr(initializer=NormalInitializer( - loc=0.0, scale=(1. / math.sqrt(hidden_dim) / 3)))) - - crf_cost = fluid.layers.linear_chain_crf( - input=emission, - label=target, - param_attr=fluid.ParamAttr( - name='crfw', - initializer=NormalInitializer( - loc=0.0, scale=(1. / math.sqrt(hidden_dim) / 3)), - learning_rate=mix_hidden_lr)) - avg_cost = fluid.layers.mean(x=crf_cost) - return avg_cost, emission - - word = fluid.layers.data(name='word', shape=[1], dtype='int64', lod_level=1) - mark = fluid.layers.data(name='mark', shape=[1], dtype='int64', lod_level=1) - target = fluid.layers.data( - name="target", shape=[1], dtype='int64', lod_level=1) - - if parallel: - places = fluid.layers.device.get_places() - pd = fluid.layers.ParallelDo(places) - with pd.do(): - word_ = pd.read_input(word) - mark_ = pd.read_input(mark) - target_ = pd.read_input(target) - avg_cost, emission_base = _net_conf(word_, mark_, target_) - pd.write_output(avg_cost) - pd.write_output(emission_base) - avg_cost_list, emission = pd() - avg_cost = fluid.layers.mean(x=avg_cost_list) - emission.stop_gradient = True - else: - avg_cost, emission = _net_conf(word, mark, target) - - return avg_cost, emission, word, mark, target diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/reader.py b/PaddleNLP/unarchived/sequence_tagging_for_ner/reader.py deleted file mode 100644 index aed2dd3d52d3bbea63e1e982468441733d44f629..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/reader.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Conll03 dataset. -""" - -from utils import * - -__all__ = ["data_reader"] - - -def canonicalize_digits(word): - if any([c.isalpha() for c in word]): return word - word = re.sub("\d", "DG", word) - if word.startswith("DG"): - word = word.replace(",", "") # remove thousands separator - return word - - -def canonicalize_word(word, wordset=None, digits=True): - word = word.lower() - if digits: - if (wordset != None) and (word in wordset): return word - word = canonicalize_digits(word) # try to canonicalize numbers - if (wordset == None) or (word in wordset): return word - else: return "UUUNKKK" # unknown token - - -def data_reader(data_file, word_dict, label_dict): - """ - The dataset can be obtained according to http://www.clips.uantwerpen.be/conll2003/ner/. - It returns a reader creator, each sample in the reader includes: - word id sequence, label id sequence and raw sentence. - - :return: reader creator - :rtype: callable - """ - - def reader(): - UNK_IDX = word_dict["UUUNKKK"] - - sentence = [] - labels = [] - with open(data_file, "r") as f: - for line in f: - if len(line.strip()) == 0: - if len(sentence) > 0: - word_idx = [ - word_dict.get( - canonicalize_word(w, word_dict), UNK_IDX) - for w in sentence - ] - mark = [1 if w[0].isupper() else 0 for w in sentence] - label_idx = [label_dict[l] for l in labels] - yield word_idx, mark, label_idx - sentence = [] - labels = [] - else: - segs = line.strip().split() - sentence.append(segs[0]) - # transform I-TYPE to BIO schema - if segs[-1] != "O" and (len(labels) == 0 or - labels[-1][1:] != segs[-1][1:]): - labels.append("B" + segs[-1][1:]) - else: - labels.append(segs[-1]) - - return reader diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/train.py b/PaddleNLP/unarchived/sequence_tagging_for_ner/train.py deleted file mode 100644 index 64a34d26d2cef52e9710ace20422e9a9a440caa1..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/train.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import print_function - -import os -import math -import time -import numpy as np -import six - -import paddle -import paddle.fluid as fluid - -import reader -from network_conf import ner_net -from utils import logger, load_dict -from utils_extend import to_lodtensor, get_embedding - - -def test(exe, chunk_evaluator, inference_program, test_data, test_fetch_list, - place): - chunk_evaluator.reset() - for data in test_data(): - word = to_lodtensor([x[0] for x in data], place) - mark = to_lodtensor([x[1] for x in data], place) - target = to_lodtensor([x[2] for x in data], place) - rets = exe.run(inference_program, - feed={"word": word, - "mark": mark, - "target": target}, - fetch_list=test_fetch_list) - num_infer = np.array(rets[0]) - num_label = np.array(rets[1]) - num_correct = np.array(rets[2]) - chunk_evaluator.update(num_infer[0].astype('int64'), - num_label[0].astype('int64'), - num_correct[0].astype('int64')) - return chunk_evaluator.eval() - - -def main(train_data_file, - test_data_file, - vocab_file, - target_file, - emb_file, - model_save_dir, - num_passes, - use_gpu, - parallel, - batch_size=200): - if not os.path.exists(model_save_dir): - os.mkdir(model_save_dir) - - word_dict = load_dict(vocab_file) - label_dict = load_dict(target_file) - - word_vector_values = get_embedding(emb_file) - - word_dict_len = len(word_dict) - label_dict_len = len(label_dict) - - if "CE_MODE_X" in os.environ: - fluid.default_startup_program().random_seed = 110 - - avg_cost, feature_out, word, mark, target = ner_net( - word_dict_len, label_dict_len, parallel) - - crf_decode = fluid.layers.crf_decoding( - input=feature_out, param_attr=fluid.ParamAttr(name='crfw')) - - (precision, recall, f1_score, num_infer_chunks, num_label_chunks, - num_correct_chunks) = fluid.layers.chunk_eval( - input=crf_decode, - label=target, - chunk_scheme="IOB", - num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0))) - chunk_evaluator = fluid.metrics.ChunkEvaluator() - - inference_program = fluid.default_main_program().clone(for_test=True) - test_fetch_list = [num_infer_chunks, num_label_chunks, num_correct_chunks] - sgd_optimizer = fluid.optimizer.SGD(learning_rate=1e-3) - sgd_optimizer.minimize(avg_cost) - - if "CE_MODE_X" not in os.environ: - train_reader = paddle.batch( - paddle.reader.shuffle( - reader.data_reader(train_data_file, word_dict, label_dict), - buf_size=20000), - batch_size=batch_size) - test_reader = paddle.batch( - paddle.reader.shuffle( - reader.data_reader(test_data_file, word_dict, label_dict), - buf_size=20000), - batch_size=batch_size) - else: - train_reader = paddle.batch( - reader.data_reader(train_data_file, word_dict, label_dict), - batch_size=batch_size) - test_reader = paddle.batch( - reader.data_reader(test_data_file, word_dict, label_dict), - batch_size=batch_size) - - place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() - feeder = fluid.DataFeeder(feed_list=[word, mark, target], place=place) - exe = fluid.Executor(place) - - exe.run(fluid.default_startup_program()) - - embedding_name = 'emb' - embedding_param = fluid.global_scope().find_var(embedding_name).get_tensor() - embedding_param.set(word_vector_values, place) - - time_begin = time.time() - for pass_id in six.moves.xrange(num_passes): - chunk_evaluator.reset() - for batch_id, data in enumerate(train_reader()): - cost_var, nums_infer, nums_label, nums_correct = exe.run( - fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[ - avg_cost, num_infer_chunks, num_label_chunks, - num_correct_chunks - ]) - if batch_id % 5 == 0: - print("Pass " + str(pass_id) + ", Batch " + str(batch_id) + - ", Cost " + str(cost_var[0])) - chunk_evaluator.update(nums_infer, nums_label, nums_correct) - pass_precision, pass_recall, pass_f1_score = chunk_evaluator.eval() - print("[TrainSet] pass_id:" + str(pass_id) + " pass_precision:" + str( - pass_precision) + " pass_recall:" + str(pass_recall) + - " pass_f1_score:" + str(pass_f1_score)) - - test_pass_precision, test_pass_recall, test_pass_f1_score = test( - exe, chunk_evaluator, inference_program, test_reader, - test_fetch_list, place) - print("[TestSet] pass_id:" + str(pass_id) + " pass_precision:" + str( - test_pass_precision) + " pass_recall:" + str(test_pass_recall) + - " pass_f1_score:" + str(test_pass_f1_score)) - - save_dirname = os.path.join(model_save_dir, "params_pass_%d" % pass_id) - if "CE_MODE_X" not in os.environ: - fluid.io.save_inference_model(save_dirname, ['word', 'mark'], - crf_decode, exe) - - if "CE_MODE_X" in os.environ: - print("kpis train_precision %f" % pass_precision) - print("kpis test_precision %f" % test_pass_precision) - print("kpis train_duration %f" % (time.time() - time_begin)) - - -if __name__ == "__main__": - main( - train_data_file="data/train", - test_data_file="data/test", - vocab_file="data/vocab.txt", - target_file="data/target.txt", - emb_file="data/wordVectors.txt", - model_save_dir="models", - num_passes=2000, - use_gpu=False, - parallel=False) diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/utils.py b/PaddleNLP/unarchived/sequence_tagging_for_ner/utils.py deleted file mode 100644 index 21567c06dbe1c0e191acadd825da1627960eff2c..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/utils.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import logging -import os -import re -import argparse -import numpy as np -from collections import defaultdict - -logger = logging.getLogger("paddle") -logger.setLevel(logging.INFO) - - -def get_embedding(emb_file='data/wordVectors.txt'): - """ - Get the trained word vector. - """ - return np.loadtxt(emb_file, dtype=float) - - -def load_dict(dict_path): - """ - Load the word dictionary from the given file. - Each line of the given file is a word, which can include multiple columns - seperated by tab. - - This function takes the first column (columns in a line are seperated by - tab) as key and takes line number of a line as the key (index of the word - in the dictionary). - """ - - return dict((line.strip().split("\t")[0], idx) - for idx, line in enumerate(open(dict_path, "r").readlines())) - - -def load_reverse_dict(dict_path): - """ - Load the word dictionary from the given file. - Each line of the given file is a word, which can include multiple columns - seperated by tab. - - This function takes line number of a line as the key (index of the word in - the dictionary) and the first column (columns in a line are seperated by - tab) as the value. - """ - return dict((idx, line.strip().split("\t")[0]) - for idx, line in enumerate(open(dict_path, "r").readlines())) diff --git a/PaddleNLP/unarchived/sequence_tagging_for_ner/utils_extend.py b/PaddleNLP/unarchived/sequence_tagging_for_ner/utils_extend.py deleted file mode 100644 index 930b19ecca5d53031e40ef40a7ec46e92799ba2a..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/sequence_tagging_for_ner/utils_extend.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import numpy as np - -import paddle.fluid as fluid - - -def get_embedding(emb_file='data/wordVectors.txt'): - """ - Get the trained word vector. - """ - return np.loadtxt(emb_file, dtype='float32') - - -def to_lodtensor(data, place): - """ - convert data to lodtensor - """ - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for l in seq_lens: - cur_len += l - lod.append(cur_len) - flattened_data = np.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - res = fluid.LoDTensor() - res.set(flattened_data, place) - res.set_lod([lod]) - return res diff --git a/PaddleNLP/unarchived/text_classification/.run_ce.sh b/PaddleNLP/unarchived/text_classification/.run_ce.sh deleted file mode 100755 index 777ba02e8c78760452a6fb7d4ac1dc4a82d62594..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/.run_ce.sh +++ /dev/null @@ -1,5 +0,0 @@ -###!/bin/bash -####This file is only used for continuous evaluation. - -export CE_MODE_X=1 -python train.py cnn | python _ce.py diff --git a/PaddleNLP/unarchived/text_classification/README.md b/PaddleNLP/unarchived/text_classification/README.md deleted file mode 100644 index 669774bac04fe906cc5bffafa1f60de60323c806..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# 文本分类 - -以下是本例的简要目录结构及说明： - -```text -. -├── nets.py # 模型定义 -├── README.md # 文档 -├── train.py # 训练脚本 -├── infer.py # 预测脚本 -└── utils.py # 定义通用函数，从外部获取 -``` - - -## 简介，模型详解 - -在PaddlePaddle v2版本[文本分类](https://github.com/PaddlePaddle/models/blob/develop/legacy/text_classification/README.md)中对于文本分类任务有较详细的介绍，在本例中不再重复介绍。 -在模型上，我们采用了bow, cnn, lstm, gru四种常见的文本分类模型。 - -## 训练 - -1. 运行命令 `python train.py bow` 开始训练模型。 - ```python - python train.py bow # bow指定网络结构，可替换成cnn, lstm, gru - ``` - -2. (可选）想自定义网络结构，需在[nets.py](./nets.py)中自行添加，并设置[train.py](./train.py)中的相应参数。 - ```python - def train(train_reader, # 训练数据 - word_dict, # 数据字典 - network, # 模型配置 - use_cuda, # 是否用GPU - parallel, # 是否并行 - save_dirname, # 保存模型路径 - lr=0.2, # 学习率大小 - batch_size=128, # 每个batch的样本数 - pass_num=30): # 训练的轮数 - ``` - -## 训练结果示例 -```text - pass_id: 0, avg_acc: 0.848040, avg_cost: 0.354073 - pass_id: 1, avg_acc: 0.914200, avg_cost: 0.217945 - pass_id: 2, avg_acc: 0.929800, avg_cost: 0.184302 - pass_id: 3, avg_acc: 0.938680, avg_cost: 0.164240 - pass_id: 4, avg_acc: 0.945120, avg_cost: 0.149150 - pass_id: 5, avg_acc: 0.951280, avg_cost: 0.137117 - pass_id: 6, avg_acc: 0.955360, avg_cost: 0.126434 - pass_id: 7, avg_acc: 0.961400, avg_cost: 0.117405 - pass_id: 8, avg_acc: 0.963560, avg_cost: 0.110070 - pass_id: 9, avg_acc: 0.965840, avg_cost: 0.103273 - pass_id: 10, avg_acc: 0.969800, avg_cost: 0.096314 - pass_id: 11, avg_acc: 0.971720, avg_cost: 0.090206 - pass_id: 12, avg_acc: 0.974800, avg_cost: 0.084970 - pass_id: 13, avg_acc: 0.977400, avg_cost: 0.078981 - pass_id: 14, avg_acc: 0.980000, avg_cost: 0.073685 - pass_id: 15, avg_acc: 0.981080, avg_cost: 0.069898 - pass_id: 16, avg_acc: 0.982080, avg_cost: 0.064923 - pass_id: 17, avg_acc: 0.984680, avg_cost: 0.060861 - pass_id: 18, avg_acc: 0.985840, avg_cost: 0.057095 - pass_id: 19, avg_acc: 0.988080, avg_cost: 0.052424 - pass_id: 20, avg_acc: 0.989160, avg_cost: 0.049059 - pass_id: 21, avg_acc: 0.990120, avg_cost: 0.045882 - pass_id: 22, avg_acc: 0.992080, avg_cost: 0.042140 - pass_id: 23, avg_acc: 0.992280, avg_cost: 0.039722 - pass_id: 24, avg_acc: 0.992840, avg_cost: 0.036607 - pass_id: 25, avg_acc: 0.994440, avg_cost: 0.034040 - pass_id: 26, avg_acc: 0.995000, avg_cost: 0.031501 - pass_id: 27, avg_acc: 0.995440, avg_cost: 0.028988 - pass_id: 28, avg_acc: 0.996240, avg_cost: 0.026639 - pass_id: 29, avg_acc: 0.996960, avg_cost: 0.024186 -``` - -## 预测 -1. 运行命令 `python infer.py bow_model`, 开始预测。 - ```python - python infer.py bow_model # bow_model指定需要导入的模型 - -## 预测结果示例 -```text - model_path: bow_model/epoch0, avg_acc: 0.882800 - model_path: bow_model/epoch1, avg_acc: 0.882360 - model_path: bow_model/epoch2, avg_acc: 0.881400 - model_path: bow_model/epoch3, avg_acc: 0.877800 - model_path: bow_model/epoch4, avg_acc: 0.872920 - model_path: bow_model/epoch5, avg_acc: 0.872640 - model_path: bow_model/epoch6, avg_acc: 0.869960 - model_path: bow_model/epoch7, avg_acc: 0.865160 - model_path: bow_model/epoch8, avg_acc: 0.863680 - model_path: bow_model/epoch9, avg_acc: 0.861200 - model_path: bow_model/epoch10, avg_acc: 0.853520 - model_path: bow_model/epoch11, avg_acc: 0.850400 - model_path: bow_model/epoch12, avg_acc: 0.855960 - model_path: bow_model/epoch13, avg_acc: 0.853480 - model_path: bow_model/epoch14, avg_acc: 0.855960 - model_path: bow_model/epoch15, avg_acc: 0.854120 - model_path: bow_model/epoch16, avg_acc: 0.854160 - model_path: bow_model/epoch17, avg_acc: 0.852240 - model_path: bow_model/epoch18, avg_acc: 0.852320 - model_path: bow_model/epoch19, avg_acc: 0.850280 - model_path: bow_model/epoch20, avg_acc: 0.849760 - model_path: bow_model/epoch21, avg_acc: 0.850160 - model_path: bow_model/epoch22, avg_acc: 0.846800 - model_path: bow_model/epoch23, avg_acc: 0.845440 - model_path: bow_model/epoch24, avg_acc: 0.845640 - model_path: bow_model/epoch25, avg_acc: 0.846200 - model_path: bow_model/epoch26, avg_acc: 0.845880 - model_path: bow_model/epoch27, avg_acc: 0.844880 - model_path: bow_model/epoch28, avg_acc: 0.844680 - model_path: bow_model/epoch29, avg_acc: 0.844960 -``` -注：过拟合导致acc持续下降，请忽略 diff --git a/PaddleNLP/unarchived/text_classification/_ce.py b/PaddleNLP/unarchived/text_classification/_ce.py deleted file mode 100644 index 6c0b1ac428d21145ab9f89ef134614b43c3db3e9..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/_ce.py +++ /dev/null @@ -1,48 +0,0 @@ -####this file is only used for continuous evaluation test! - -import os -import sys -sys.path.append(os.environ['ceroot']) -from kpi import CostKpi, DurationKpi, AccKpi - -#### NOTE kpi.py should shared in models in some way!!!! - -train_acc_kpi = AccKpi('train_acc', 0.005, actived=True) -train_cost_kpi = CostKpi('train_cost', 0.005, actived=False) -train_duration_kpi = DurationKpi('train_duration', 0.05, actived=True) - -tracking_kpis = [ - train_acc_kpi, - train_cost_kpi, - train_duration_kpi, -] - - -def parse_log(log): - for line in log.split('\n'): - fs = line.strip().split('\t') - print(fs) - if len(fs) == 3 and fs[0] == 'kpis': - print("-----%s" % fs) - kpi_name = fs[1] - kpi_value = float(fs[2]) - yield kpi_name, kpi_value - - -def log_to_ce(log): - kpi_tracker = {} - for kpi in tracking_kpis: - kpi_tracker[kpi.name] = kpi - - for (kpi_name, kpi_value) in parse_log(log): - print(kpi_name, kpi_value) - kpi_tracker[kpi_name].add_record(kpi_value) - kpi_tracker[kpi_name].persist() - - -if __name__ == '__main__': - log = sys.stdin.read() - print("*****") - print(log) - print("****") - log_to_ce(log) diff --git a/PaddleNLP/unarchived/text_classification/async_executor/README.md b/PaddleNLP/unarchived/text_classification/async_executor/README.md deleted file mode 100644 index 0e36a8be7653787852d4d04b7603cec1046f61be..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/async_executor/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# 文本分类 - -以下是本例的简要目录结构及说明： - -```text -. -|-- README.md # README -|-- data_generator # IMDB数据集生成工具 -| |-- IMDB.py # 在data_generator.py基础上扩展IMDB数据集处理逻辑 -| |-- build_raw_data.py # IMDB数据预处理，其产出被splitfile.py读取。格式：word word ... | label -| |-- data_generator.py # 与AsyncExecutor配套的数据生成工具框架 -| `-- splitfile.py # 将build_raw_data.py生成的文件切分，其产出被IMDB.py读取 -|-- data_generator.sh # IMDB数据集生成工具入口 -|-- data_reader.py # 预测脚本使用的数据读取工具 -|-- infer.py # 预测脚本 -`-- train.py # 训练脚本 -``` - -## 简介 - -本目录包含用fluid.AsyncExecutor训练文本分类任务的脚本。网络模型定义沿用自父目录nets.py - -## 训练 - -1. 运行命令 `sh data_generator.sh`，下载IMDB数据集，并转化成适合AsyncExecutor读取的训练数据 -2. 运行命令 `python train.py bow` 开始训练模型。 - ```python - python train.py bow # bow指定网络结构，可替换成cnn, lstm, gru - ``` - -3. (可选）想自定义网络结构，需在[nets.py](../nets.py)中自行添加，并设置[train.py](./train.py)中的相应参数。 - ```python - def train(train_reader, # 训练数据 - word_dict, # 数据字典 - network, # 模型配置 - use_cuda, # 是否用GPU - parallel, # 是否并行 - save_dirname, # 保存模型路径 - lr=0.2, # 学习率大小 - batch_size=128, # 每个batch的样本数 - pass_num=30): # 训练的轮数 - ``` - -## 训练结果示例 - -```text -pass_id: 0 pass_time_cost 4.723438 -pass_id: 1 pass_time_cost 3.867186 -pass_id: 2 pass_time_cost 4.490111 -pass_id: 3 pass_time_cost 4.573296 -pass_id: 4 pass_time_cost 4.180547 -pass_id: 5 pass_time_cost 4.214476 -pass_id: 6 pass_time_cost 4.520387 -pass_id: 7 pass_time_cost 4.149485 -pass_id: 8 pass_time_cost 3.821354 -pass_id: 9 pass_time_cost 5.136178 -pass_id: 10 pass_time_cost 4.137318 -pass_id: 11 pass_time_cost 3.943429 -pass_id: 12 pass_time_cost 3.766478 -pass_id: 13 pass_time_cost 4.235983 -pass_id: 14 pass_time_cost 4.796462 -pass_id: 15 pass_time_cost 4.668116 -pass_id: 16 pass_time_cost 4.373798 -pass_id: 17 pass_time_cost 4.298131 -pass_id: 18 pass_time_cost 4.260021 -pass_id: 19 pass_time_cost 4.244411 -pass_id: 20 pass_time_cost 3.705138 -pass_id: 21 pass_time_cost 3.728070 -pass_id: 22 pass_time_cost 3.817919 -pass_id: 23 pass_time_cost 4.698598 -pass_id: 24 pass_time_cost 4.859262 -pass_id: 25 pass_time_cost 5.725732 -pass_id: 26 pass_time_cost 5.102599 -pass_id: 27 pass_time_cost 3.876582 -pass_id: 28 pass_time_cost 4.762538 -pass_id: 29 pass_time_cost 3.797759 -``` -与fluid.Executor不同，AsyncExecutor在每个pass结束不会将accuracy打印出来。为了观察训练过程，可以将fluid.AsyncExecutor.run()方法的Debug参数设为True，这样每个pass结束会把参数指定的fetch variable打印出来： - -``` -async_executor.run( - main_program, - dataset, - filelist, - thread_num, - [acc], - debug=True) -``` - -## 预测 - -1. 运行命令 `python infer.py bow_model`, 开始预测。 - ```python - python infer.py bow_model # bow_model指定需要导入的模型 - ``` - -## 预测结果示例 -```text -model_path: bow_model/epoch0.model, avg_acc: 0.882600 -model_path: bow_model/epoch1.model, avg_acc: 0.887920 -model_path: bow_model/epoch2.model, avg_acc: 0.886920 -model_path: bow_model/epoch3.model, avg_acc: 0.884720 -model_path: bow_model/epoch4.model, avg_acc: 0.879760 -model_path: bow_model/epoch5.model, avg_acc: 0.876920 -model_path: bow_model/epoch6.model, avg_acc: 0.874160 -model_path: bow_model/epoch7.model, avg_acc: 0.872000 -model_path: bow_model/epoch8.model, avg_acc: 0.870360 -model_path: bow_model/epoch9.model, avg_acc: 0.868480 -model_path: bow_model/epoch10.model, avg_acc: 0.867240 -model_path: bow_model/epoch11.model, avg_acc: 0.866200 -model_path: bow_model/epoch12.model, avg_acc: 0.865560 -model_path: bow_model/epoch13.model, avg_acc: 0.865160 -model_path: bow_model/epoch14.model, avg_acc: 0.864480 -model_path: bow_model/epoch15.model, avg_acc: 0.864240 -model_path: bow_model/epoch16.model, avg_acc: 0.863800 -model_path: bow_model/epoch17.model, avg_acc: 0.863520 -model_path: bow_model/epoch18.model, avg_acc: 0.862760 -model_path: bow_model/epoch19.model, avg_acc: 0.862680 -model_path: bow_model/epoch20.model, avg_acc: 0.862240 -model_path: bow_model/epoch21.model, avg_acc: 0.862280 -model_path: bow_model/epoch22.model, avg_acc: 0.862080 -model_path: bow_model/epoch23.model, avg_acc: 0.861560 -model_path: bow_model/epoch24.model, avg_acc: 0.861280 -model_path: bow_model/epoch25.model, avg_acc: 0.861160 -model_path: bow_model/epoch26.model, avg_acc: 0.861080 -model_path: bow_model/epoch27.model, avg_acc: 0.860920 -model_path: bow_model/epoch28.model, avg_acc: 0.860800 -model_path: bow_model/epoch29.model, avg_acc: 0.860760 -``` -注：过拟合导致acc持续下降，请忽略 diff --git a/PaddleNLP/unarchived/text_classification/async_executor/data_generator.sh b/PaddleNLP/unarchived/text_classification/async_executor/data_generator.sh deleted file mode 100644 index 7b5aad5f7609924b469d29cad8c95c7df8e75b6e..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/async_executor/data_generator.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pushd . -cd ./data_generator - -# wget "http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz" -if [ ! -f aclImdb_v1.tar.gz ]; then - wget "http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz" -fi -tar zxvf aclImdb_v1.tar.gz - -mkdir train_data -python build_raw_data.py train | python splitfile.py 12 train_data - -mkdir test_data -python build_raw_data.py test | python splitfile.py 12 test_data - -/opt/python27/bin/python IMDB.py train_data -/opt/python27/bin/python IMDB.py test_data - -mv ./output_dataset/train_data ../ -mv ./output_dataset/test_data ../ -cp aclImdb/imdb.vocab ../ - -rm -rf ./output_dataset -rm -rf train_data -rm -rf test_data -rm -rf aclImdb -popd diff --git a/PaddleNLP/unarchived/text_classification/async_executor/data_generator/IMDB.py b/PaddleNLP/unarchived/text_classification/async_executor/data_generator/IMDB.py deleted file mode 100644 index 579df4e0e722d245cabc366ffaeeab71dbf2aa0a..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/async_executor/data_generator/IMDB.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import os, sys -sys.path.append(os.path.abspath(os.path.join('..'))) -from data_generator import MultiSlotDataGenerator - - -class IMDbDataGenerator(MultiSlotDataGenerator): - def load_resource(self, dictfile): - self._vocab = {} - wid = 0 - with open(dictfile) as f: - for line in f: - self._vocab[line.strip()] = wid - wid += 1 - self._unk_id = len(self._vocab) - self._pattern = re.compile(r'(;|,|\.|\?|!|\s|\(|\))') - - def process(self, line): - send = '|'.join(line.split('|')[:-1]).lower().replace("
", - " ").strip() - label = [int(line.split('|')[-1])] - - words = [x for x in self._pattern.split(send) if x and x != " "] - feas = [ - self._vocab[x] if x in self._vocab else self._unk_id for x in words - ] - - return ("words", feas), ("label", label) - - -imdb = IMDbDataGenerator() -imdb.load_resource("aclImdb/imdb.vocab") - -# data from files -file_names = os.listdir(sys.argv[1]) -filelist = [] -for i in range(0, len(file_names)): - filelist.append(os.path.join(sys.argv[1], file_names[i])) - -line_limit = 2500 -process_num = 24 -imdb.run_from_files( - filelist=filelist, - line_limit=line_limit, - process_num=process_num, - output_dir=('output_dataset/%s' % (sys.argv[1]))) diff --git a/PaddleNLP/unarchived/text_classification/async_executor/data_generator/build_raw_data.py b/PaddleNLP/unarchived/text_classification/async_executor/data_generator/build_raw_data.py deleted file mode 100644 index 2c0c0981c93b3b1e9231c7efe1f0b49e178c060f..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/async_executor/data_generator/build_raw_data.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Build raw data -""" -from __future__ import print_function -import sys -import os -import random -import re -data_type = sys.argv[1] - -if not (data_type == "train" or data_type == "test"): - print("python %s [test/train]" % sys.argv[0], file=sys.stderr) - sys.exit(-1) - -pos_folder = "aclImdb/" + data_type + "/pos/" -neg_folder = "aclImdb/" + data_type + "/neg/" - -pos_train_list = [(pos_folder + x, "1") for x in os.listdir(pos_folder)] -neg_train_list = [(neg_folder + x, "0") for x in os.listdir(neg_folder)] - -all_train_list = pos_train_list + neg_train_list -random.shuffle(all_train_list) - - -def load_dict(dictfile): - """ - Load word id dict - """ - vocab = {} - wid = 0 - with open(dictfile) as f: - for line in f: - vocab[line.strip()] = str(wid) - wid += 1 - return vocab - - -vocab = load_dict("aclImdb/imdb.vocab") -unk_id = str(len(vocab)) -print("vocab size: ", len(vocab), file=sys.stderr) -pattern = re.compile(r'(;|,|\.|\?|!|\s|\(|\))') - -for fitem in all_train_list: - label = str(fitem[1]) - fname = fitem[0] - with open(fname) as f: - sent = f.readline().lower().replace("
", " ").strip() - out_s = "%s | %s" % (sent, label) - print(out_s, file=sys.stdout) diff --git a/PaddleNLP/unarchived/text_classification/async_executor/data_generator/data_generator.py b/PaddleNLP/unarchived/text_classification/async_executor/data_generator/data_generator.py deleted file mode 100644 index 70d1e1f9a020be13f43129cf26964c860ae2ce4f..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/async_executor/data_generator/data_generator.py +++ /dev/null @@ -1,508 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import multiprocessing -__all__ = ['MultiSlotDataGenerator'] - - -class DataGenerator(object): - def __init__(self): - self._proto_info = None - - def _set_filelist(self, filelist): - if not isinstance(filelist, list) and not isinstance(filelist, tuple): - raise ValueError("filelist%s must be in list or tuple type" % - type(filelist)) - if not filelist: - raise ValueError("filelist can not be empty") - self._filelist = filelist - - def _set_process_num(self, process_num): - if not isinstance(process_num, int): - raise ValueError("process_num%s must be in int type" % - type(process_num)) - if process_num < 1: - raise ValueError("process_num can not less than 1") - self._process_num = process_num - - def _set_line_limit(self, line_limit): - if not isinstance(line_limit, int): - raise ValueError("line_limit%s must be in int type" % - type(line_limit)) - if line_limit < 1: - raise ValueError("line_limit can not less than 1") - self._line_limit = line_limit - - def _set_output_dir(self, output_dir): - if not isinstance(output_dir, str): - raise ValueError("output_dir%s must be in str type" % - type(output_dir)) - if not output_dir: - raise ValueError("output_dir can not be empty") - self._output_dir = output_dir - - def _set_output_prefix(self, output_prefix): - if not isinstance(output_prefix, str): - raise ValueError("output_prefix%s must be in str type" % - type(output_prefix)) - self._output_prefix = output_prefix - - def _set_output_fill_digit(self, output_fill_digit): - if not isinstance(output_fill_digit, int): - raise ValueError("output_fill_digit%s must be in int type" % - type(output_fill_digit)) - if output_fill_digit < 1: - raise ValueError("output_fill_digit can not less than 1") - self._output_fill_digit = output_fill_digit - - def _set_proto_filename(self, proto_filename): - if not isinstance(proto_filename, str): - raise ValueError("proto_filename%s must be in str type" % - type(proto_filename)) - if not proto_filename: - raise ValueError("proto_filename can not be empty") - self._proto_filename = proto_filename - - def _print_info(self): - ''' - Print the configuration information - (Called only in the run_from_stdin function). - ''' - sys.stderr.write("=" * 16 + " config " + "=" * 16 + "\n") - sys.stderr.write(" filelist size: %d\n" % len(self._filelist)) - sys.stderr.write(" process num: %d\n" % self._process_num) - sys.stderr.write(" line limit: %d\n" % self._line_limit) - sys.stderr.write(" output dir: %s\n" % self._output_dir) - sys.stderr.write(" output prefix: %s\n" % self._output_prefix) - sys.stderr.write(" output fill digit: %d\n" % self._output_fill_digit) - sys.stderr.write(" proto filename: %s\n" % self._proto_filename) - sys.stderr.write("==== This may take a few minutes... ====\n") - - def _get_output_filename(self, output_index, lock=None): - ''' - This function is used to get the name of the output file and - update output_index. - Args: - output_index(manager.Value(i)): the index of output file. - lock(manager.Lock): The lock for processes safe. - Return: - Return the name(string) of output file. - ''' - if lock is not None: lock.acquire() - file_index = output_index.value - output_index.value += 1 - if lock is not None: lock.release() - filename = os.path.join(self._output_dir, self._output_prefix) \ - + str(file_index).zfill(self._output_fill_digit) - sys.stderr.write("[%d] write data to file: %s\n" % - (os.getpid(), filename)) - return filename - - def run_from_stdin(self, - is_local=True, - hadoop_host=None, - hadoop_ugi=None, - proto_path=None, - proto_filename="data_feed.proto"): - ''' - This function reads the data row from stdin, parses it with the - process function, and further parses the return value of the - process function with the _gen_str function. The parsed data will - be wrote to stdout and the corresponding protofile will be - generated. If local is set to False, the protofile will be - uploaded to hadoop. - Args: - is_local(bool): Whether to execute locally. If it is False, the - protofile will be uploaded to hadoop. The - default value is True. - hadoop_host(str): The host name of the hadoop. It should be - in this format: "hdfs://${HOST}:${PORT}". - hadoop_ugi(str): The ugi of the hadoop. It should be in this - format: "${USERNAME},${PASSWORD}". - proto_path(str): The hadoop path you want to upload the - protofile to. - proto_filename(str): The name of protofile. The default value - is "data_feed.proto". It is not - recommended to modify it. - ''' - if is_local: - print \ -'''\033[1;34m======================================================= - Pay attention to that the version of Python in Hadoop - may inconsistent with local version. Please check the - Python version of Hadoop to ensure that it is >= 2.7. -=======================================================\033[0m''' - else: - if hadoop_ugi is None or \ - hadoop_host is None or \ - proto_path is None: - raise ValueError( - "pls set hadoop_ugi, hadoop_host, and proto_path") - self._set_proto_filename(proto_filename) - for line in sys.stdin: - user_parsed_line = self.process(line) - sys.stdout.write(self._gen_str(user_parsed_line)) - if self._proto_info is not None: - # maybe some task do not catch files - with open(self._proto_filename, "w") as f: - f.write(self._get_proto_desc(self._proto_info)) - if is_local == False: - cmd = "$HADOOP_HOME/bin/hadoop fs" \ - + " -Dhadoop.job.ugi=" + hadoop_ugi \ - + " -Dfs.default.name=" + hadoop_host \ - + " -put " + self._proto_filename + " " + proto_path - os.system(cmd) - - def run_from_files(self, - filelist, - line_limit, - process_num=1, - output_dir="./output_dataset", - output_prefix="part-", - output_fill_digit=8, - proto_filename="data_feed.proto"): - ''' - This function will run process_num processes to process the files - in the filelist. It will create the output data folder(output_dir) - in the current directory, and write the processed data into the - output_dir folder(each file line_limit data, the prefix of filename - is output_prefix, the suffix of filename is output_fill_digit - numbers). And the proto_info is generated at the same time. the - name of proto file will be proto_filename. - Args: - filelist(list or tuple): Files that need to be processed. - line_limit(int): Maximum number of data stored per file. - process_num(int): Number of processes running simultaneously. - output_dir(str): The name of the folder where the output - data file is stored. - output_prefix(str): The prefix of output data file. - output_fill_digit(int): The number of suffix numbers of the - output data file. - proto_filename(str): The name of protofile. - ''' - self._set_filelist(filelist) - self._set_line_limit(line_limit) - self._set_process_num(min(process_num, len(filelist))) - self._set_output_dir(output_dir) - self._set_output_prefix(output_prefix) - self._set_output_fill_digit(output_fill_digit) - self._set_proto_filename(proto_filename) - self._print_info() - - if not os.path.exists(self._output_dir): - os.makedirs(self._output_dir) - elif not os.path.isdir(self._output_dir): - raise ValueError("%s is not a directory" % self._output_dir) - - processes = multiprocessing.Pool() - manager = multiprocessing.Manager() - output_index = manager.Value('i', 0) - file_queue = manager.Queue() - lock = manager.Lock() - remaining_queue = manager.Queue() - for file in self._filelist: - file_queue.put(file) - info_result = [] - for i in range(self._process_num): - info_result.append(processes.apply_async(subprocess_wrapper, \ - (self, file_queue, remaining_queue, output_index, lock, ))) - processes.close() - processes.join() - - infos = [ - result.get() for result in info_result if result.get() is not None - ] - proto_info = self._combine_infos(infos) - with open(os.path.join(self._output_dir, self._proto_filename), - "w") as f: - f.write(self._get_proto_desc(proto_info)) - - while not remaining_queue.empty(): - with open(self._get_output_filename(output_index), "w") as f: - for i in range(min(self._line_limit, remaining_queue.qsize())): - f.write(remaining_queue.get(False)) - - def _subprocess(self, file_queue, remaining_queue, output_index, lock): - ''' - This function will be called by multiple processes. It is used to - continuously fetch files from file_queue, using process() function - (defined by user) and _gen_str() function(defined by concrete classes) - to process data in units of rows. Write the processed data to the - file(each file will be self._line_limit line). If the file in the - file_queue has been consumed, but the file is not full, the data - that is less than the self._line_limit line will be stored in the - remaining_queue. - Args: - file_queue(manager.Queue): The queue contains all the file - names to be processed. - remaining_queue(manager.Queue): The queue contains the data that - is less than the self._line_limit - line. - output_index(manager.Value(i)): The index(suffix) of the - output file. - lock(manager.Lock): The lock for processes safe. - Returns: - Return a proto_info which can be translated into a proto string. - ''' - buffer = [] - while not file_queue.empty(): - try: - filename = file_queue.get(False) - except: # file_queue empty - break - with open(filename, 'r') as f: - for line in f: - buffer.append(self._gen_str(self.process(line))) - if len(buffer) == self._line_limit: - with open( - self._get_output_filename(output_index, lock), - "w") as wf: - for x in buffer: - wf.write(x) - buffer = [] - if buffer: - for x in buffer: - remaining_queue.put(x) - return self._proto_info - - def _gen_str(self, line): - ''' - Further processing the output of the process() function rewritten by - user, outputting data that can be directly read by the datafeed,and - updating proto_info infomation. - Args: - line(str): the output of the process() function rewritten by user. - Returns: - Return a string data that can be read directly by the datafeed. - ''' - raise NotImplementedError( - "pls use MultiSlotDataGenerator or PairWiseDataGenerator") - - def _combine_infos(self, infos): - ''' - This function is used to merge proto_info information from different - processes. In general, the proto_info of each process is consistent. - Args: - infos(list): the list of proto_infos from different processes. - Returns: - Return a unified proto_info. - ''' - raise NotImplementedError( - "pls use MultiSlotDataGenerator or PairWiseDataGenerator") - - def _get_proto_desc(self, proto_info): - ''' - This function outputs the string of the proto file(can be directly - written to the file) according to the proto_info information. - Args: - proto_info: The proto information used to generate the proto - string. The type of the variable will be determined - by the subclass. In the MultiSlotDataGenerator, - proto_info variable is a list of tuple. - Returns: - Returns a string of the proto file. - ''' - raise NotImplementedError( - "pls use MultiSlotDataGenerator or PairWiseDataGenerator") - - def process(self, line): - ''' - This function needs to be overridden by the user to process the - original data row into a list or tuple. - Args: - line(str): the original data row - Returns: - Returns the data processed by the user. - The data format is list or tuple: - [(name, [feasign, ...]), ...] - or ((name, [feasign, ...]), ...) - - For example: - [("words", [1926, 08, 17]), ("label", [1])] - or (("words", [1926, 08, 17]), ("label", [1])) - Note: - The type of feasigns must be in int or float. Once the float - element appears in the feasign, the type of that slot will be - processed into a float. - ''' - raise NotImplementedError( - "pls rewrite this function to return a list or tuple: " + - "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)") - - -def subprocess_wrapper(instance, file_queue, remaining_queue, output_index, - lock): - ''' - In order to use the class function as a process, you need to wrap it. - ''' - return instance._subprocess(file_queue, remaining_queue, output_index, lock) - - -class MultiSlotDataGenerator(DataGenerator): - def _combine_infos(self, infos): - ''' - This function is used to merge proto_info information from different - processes. In general, the proto_info of each process is consistent. - The type of input infos is list, and the type of element of infos is - tuple. The format of element of infos will be (name, type). - Args: - infos(list): the list of proto_infos from different processes. - Returns: - Return a unified proto_info. - Note: - This function is only called by the run_from_files function, so - when using the run_from_stdin function(usually used for hadoop), - the output of the process function(rewritten by the user) does - not allow that the same field to have both float and int type - values. - ''' - proto_info = infos[0] - for info in infos: - for index, slot in enumerate(info): - name, type = slot - if name != proto_info[index][0]: - raise ValueError( - "combine infos error, pls contact the maintainer of this code~" - ) - if type == "float" and proto_info[index][1] == "uint64": - proto_info[index] = (name, type) - return proto_info - - def _get_proto_desc(self, proto_info): - ''' - Generate a string of proto file based on the proto_info information. - - The proto_info will be a list of tuples: - >>> [(Name, Type), ...] - - The string of proto file will be in this format: - >>> name: "MultiSlotDataFeed" - >>> batch_size: 32 - >>> multi_slot_desc { - >>> slots { - >>> name: Name - >>> type: Type - >>> is_dense: false - >>> is_used: false - >>> } - >>> } - Args: - proto_info(list): The proto information used to generate the - proto string. - Returns: - Returns a string of the proto file. - ''' - proto_str = "name: \"MultiSlotDataFeed\"\n" \ - + "batch_size: 32\nmulti_slot_desc {\n" - for elem in proto_info: - proto_str += " slots {\n" \ - + " name: \"%s\"\n" % elem[0]\ - + " type: \"%s\"\n" % elem[1]\ - + " is_dense: false\n" \ - + " is_used: false\n" \ - + " }\n" - proto_str += "}" - return proto_str - - def _gen_str(self, line): - ''' - Further processing the output of the process() function rewritten by - user, outputting data that can be directly read by the MultiSlotDataFeed, - and updating proto_info infomation. - The input line will be in this format: - >>> [(name, [feasign, ...]), ...] - >>> or ((name, [feasign, ...]), ...) - The output will be in this format: - >>> [ids_num id1 id2 ...] ... - The proto_info will be in this format: - >>> [(name, type), ...] - - For example, if the input is like this: - >>> [("words", [1926, 08, 17]), ("label", [1])] - >>> or (("words", [1926, 08, 17]), ("label", [1])) - the output will be: - >>> 3 1234 2345 3456 1 1 - the proto_info will be: - >>> [("words", "uint64"), ("label", "uint64")] - Args: - line(str): the output of the process() function rewritten by user. - Returns: - Return a string data that can be read directly by the MultiSlotDataFeed. - ''' - if not isinstance(line, list) and not isinstance(line, tuple): - raise ValueError( - "the output of process() must be in list or tuple type") - output = "" - - if self._proto_info is None: - self._proto_info = [] - for item in line: - name, elements = item - if not isinstance(name, str): - raise ValueError("name%s must be in str type" % type(name)) - if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) - if not elements: - raise ValueError( - "the elements of each field can not be empty, you need padding it in process()." - ) - self._proto_info.append((name, "uint64")) - if output: - output += " " - output += str(len(elements)) - for elem in elements: - if isinstance(elem, float): - self._proto_info[-1] = (name, "float") - elif not isinstance(elem, int) and not isinstance(elem, - long): - raise ValueError( - "the type of element%s must be in int or float" % - type(elem)) - output += " " + str(elem) - else: - if len(line) != len(self._proto_info): - raise ValueError( - "the complete field set of two given line are inconsistent.") - for index, item in enumerate(line): - name, elements = item - if not isinstance(name, str): - raise ValueError("name%s must be in str type" % type(name)) - if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) - if not elements: - raise ValueError( - "the elements of each field can not be empty, you need padding it in process()." - ) - if name != self._proto_info[index][0]: - raise ValueError( - "the field name of two given line are not match: require<%s>, get<%d>." - % (self._proto_info[index][0], name)) - if output: - output += " " - output += str(len(elements)) - for elem in elements: - if self._proto_info[index][1] != "float": - if isinstance(elem, float): - self._proto_info[index] = (name, "float") - elif not isinstance(elem, int) and not isinstance(elem, - long): - raise ValueError( - "the type of element%s must be in int or float" - % type(elem)) - output += " " + str(elem) - return output + "\n" diff --git a/PaddleNLP/unarchived/text_classification/async_executor/data_generator/splitfile.py b/PaddleNLP/unarchived/text_classification/async_executor/data_generator/splitfile.py deleted file mode 100644 index 414e097c5a6352673c00e487230e38bf64e6299a..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/async_executor/data_generator/splitfile.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Split file into parts -""" -import sys -import os -block = int(sys.argv[1]) -datadir = sys.argv[2] -file_list = [] -for i in range(block): - file_list.append(open(datadir + "/part-" + str(i), "w")) -id_ = 0 -for line in sys.stdin: - file_list[id_ % block].write(line) - id_ += 1 -for f in file_list: - f.close() diff --git a/PaddleNLP/unarchived/text_classification/async_executor/data_reader.py b/PaddleNLP/unarchived/text_classification/async_executor/data_reader.py deleted file mode 100644 index 0cdad76295ea7d6d2eb7f2982aa5cb6b830d4976..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/async_executor/data_reader.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import os -import paddle - - -def parse_fields(fields): - words_width = int(fields[0]) - words = fields[1:1 + words_width] - label = fields[-1] - - return words, label - - -def imdb_data_feed_reader(data_dir, batch_size, buf_size): - """ - Data feed reader for IMDB dataset. - This data set has been converted from original format to a format suitable - for AsyncExecutor - See data.proto for data format - """ - - def reader(): - for file in os.listdir(data_dir): - if file.endswith('.proto'): - continue - - with open(os.path.join(data_dir, file), 'r') as f: - for line in f: - fields = line.split(' ') - words, label = parse_fields(fields) - yield words, label - - test_reader = paddle.batch( - paddle.reader.shuffle( - reader, buf_size=buf_size), batch_size=batch_size) - return test_reader diff --git a/PaddleNLP/unarchived/text_classification/async_executor/infer.py b/PaddleNLP/unarchived/text_classification/async_executor/infer.py deleted file mode 100644 index 07a167c76c150027849571602856d5eeeabcf028..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/async_executor/infer.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import unittest -import contextlib -import numpy as np - -import paddle -import paddle.fluid as fluid - -import data_reader - - -def infer(test_reader, use_cuda, model_path=None): - """ - inference function - """ - if model_path is None: - print(str(model_path) + " cannot be found") - return - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - - inference_scope = fluid.Scope() - with fluid.scope_guard(inference_scope): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(model_path, exe) - - total_acc = 0.0 - total_count = 0 - for data in test_reader(): - acc = exe.run(inference_program, - feed=utils.data2tensor(data, place), - fetch_list=fetch_targets, - return_numpy=True) - total_acc += acc[0] * len(data) - total_count += len(data) - - avg_acc = total_acc / total_count - print("model_path: %s, avg_acc: %f" % (model_path, avg_acc)) - - -if __name__ == "__main__": - if __package__ is None: - from os import sys, path - sys.path.append( - os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - import utils - - batch_size = 128 - model_path = sys.argv[1] - test_data_dirname = 'test_data' - - if len(sys.argv) == 3: - test_data_dirname = sys.argv[2] - - test_reader = data_reader.imdb_data_feed_reader( - 'test_data', batch_size, buf_size=500000) - - models = os.listdir(model_path) - for i in range(0, len(models)): - epoch_path = "epoch" + str(i) + ".model" - epoch_path = os.path.join(model_path, epoch_path) - infer(test_reader, use_cuda=False, model_path=epoch_path) diff --git a/PaddleNLP/unarchived/text_classification/async_executor/train.py b/PaddleNLP/unarchived/text_classification/async_executor/train.py deleted file mode 100644 index 034d65dd5bf94a717f791e04b8648d9606528d6c..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/async_executor/train.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import multiprocessing - -import paddle -import paddle.fluid as fluid - - -def train(network, dict_dim, lr, save_dirname, training_data_dirname, pass_num, - thread_num, batch_size): - file_names = os.listdir(training_data_dirname) - filelist = [] - for i in range(0, len(file_names)): - if file_names[i] == 'data_feed.proto': - continue - filelist.append(os.path.join(training_data_dirname, file_names[i])) - - dataset = fluid.DataFeedDesc( - os.path.join(training_data_dirname, 'data_feed.proto')) - dataset.set_batch_size( - batch_size) # datafeed should be assigned a batch size - dataset.set_use_slots(['words', 'label']) - - data = fluid.layers.data( - name="words", shape=[1], dtype="int64", lod_level=1) - label = fluid.layers.data(name="label", shape=[1], dtype="int64") - - avg_cost, acc, prediction = network(data, label, dict_dim) - optimizer = fluid.optimizer.Adagrad(learning_rate=lr) - opt_ops, weight_and_grad = optimizer.minimize(avg_cost) - - startup_program = fluid.default_startup_program() - main_program = fluid.default_main_program() - - place = fluid.CPUPlace() - executor = fluid.Executor(place) - executor.run(startup_program) - - async_executor = fluid.AsyncExecutor(place) - for i in range(pass_num): - pass_start = time.time() - async_executor.run(main_program, - dataset, - filelist, - thread_num, [acc], - debug=False) - print('pass_id: %u pass_time_cost %f' % (i, time.time() - pass_start)) - fluid.io.save_inference_model('%s/epoch%d.model' % (save_dirname, i), - [data.name, label.name], [acc], executor) - - -if __name__ == "__main__": - if __package__ is None: - from os import sys, path - sys.path.append( - os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - - from nets import bow_net, cnn_net, lstm_net, gru_net - from utils import load_vocab - - batch_size = 4 - lr = 0.002 - pass_num = 30 - save_dirname = "" - thread_num = multiprocessing.cpu_count() - - if sys.argv[1] == "bow": - network = bow_net - batch_size = 128 - save_dirname = "bow_model" - elif sys.argv[1] == "cnn": - network = cnn_net - lr = 0.01 - save_dirname = "cnn_model" - elif sys.argv[1] == "lstm": - network = lstm_net - lr = 0.05 - save_dirname = "lstm_model" - elif sys.argv[1] == "gru": - network = gru_net - batch_size = 128 - lr = 0.05 - save_dirname = "gru_model" - - training_data_dirname = 'train_data/' - if len(sys.argv) == 3: - training_data_dirname = sys.argv[2] - - if len(sys.argv) == 4: - if thread_num >= int(sys.argv[3]): - thread_num = int(sys.argv[3]) - - vocab = load_vocab('imdb.vocab') - dict_dim = len(vocab) - - train(network, dict_dim, lr, save_dirname, training_data_dirname, pass_num, - thread_num, batch_size) diff --git a/PaddleNLP/unarchived/text_classification/clouds/scdb_parallel_executor.py b/PaddleNLP/unarchived/text_classification/clouds/scdb_parallel_executor.py deleted file mode 100644 index 7a3cbb11365c5d7cbff34c27d1b3d4c28f9fe401..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/clouds/scdb_parallel_executor.py +++ /dev/null @@ -1,503 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest -import contextlib -import paddle -import paddle.fluid as fluid -import numpy as np -import six -import sys -import time -import os -import json -import random - - -def to_lodtensor(data, place): - """ - convert to LODtensor - """ - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for l in seq_lens: - cur_len += l - lod.append(cur_len) - flattened_data = np.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - res = fluid.LoDTensor() - res.set(flattened_data, place) - res.set_lod([lod]) - return res - - -def load_vocab(filename): - """ - load imdb vocabulary - """ - vocab = {} - with open(filename) as f: - wid = 0 - for line in f: - vocab[line.strip()] = wid - wid += 1 - vocab[""] = len(vocab) - return vocab - - -def data2tensor(data, place): - """ - data2tensor - """ - input_seq = to_lodtensor([x[0] for x in data], place) - y_data = np.array([x[1] for x in data]).astype("int64") - y_data = y_data.reshape([-1, 1]) - return {"words": input_seq, "label": y_data} - - -def data2pred(data, place): - """ - data2tensor - """ - input_seq = to_lodtensor([x[0] for x in data], place) - y_data = np.array([x[1] for x in data]).astype("int64") - y_data = y_data.reshape([-1, 1]) - return {"words": input_seq} - - -def load_dict(vocab): - """ - Load dict from vocab - """ - word_dict = dict() - with open(vocab, "r") as fin: - for line in fin: - cols = line.strip("\r\n").decode("gb18030").split("\t") - word_dict[cols[0]] = int(cols[1]) - return word_dict - - -def save_dict(word_dict, vocab): - """ - Save dict into file - """ - with open(vocab, "w") as fout: - for k, v in six.iteritems(word_dict): - outstr = ("%s\t%s\n" % (k, v)).encode("gb18030") - fout.write(outstr) - - -def build_dict(fname): - """ - build word dict using trainset - """ - word_dict = dict() - with open(fname, "r") as fin: - for line in fin: - try: - words = line.strip("\r\n").decode("gb18030").split("\t")[ - 1].split(" ") - except: - sys.stderr.write("[warning] build_dict: decode error\n") - continue - for w in words: - if w not in word_dict: - word_dict[w] = len(word_dict) - return word_dict - - -def scdb_word_dict(vocab="scdb_data/train_set/train.vocab"): - """ - get word_dict - """ - if not os.path.exists(vocab): - w_dict = build_dict(train_file) - save_dict(w_dict, vocab) - else: - w_dict = load_dict(vocab) - w_dict[""] = len(w_dict) - return w_dict - - -def data_reader(fname, word_dict, is_dir=False): - """ - Convert word sequence into slot - """ - unk_id = len(word_dict) - all_data = [] - filelist = [] - if is_dir: - filelist = [fname + os.sep + f for f in os.listdir(fname)] - else: - filelist = [fname] - - for each_name in filelist: - with open(each_name, "r") as fin: - for line in fin: - try: - cols = line.strip("\r\n").decode("gb18030").split("\t") - except: - sys.stderr.write("warning: ignore decode error\n") - continue - - label = int(cols[0]) - wids = [ - word_dict[x] if x in word_dict else unk_id - for x in cols[1].split(" ") - ] - all_data.append((wids, label)) - - random.shuffle(all_data) - - def reader(): - for doc, label in all_data: - yield doc, label - - return reader - - -def scdb_train_data(train_dir="scdb_data/train_set/corpus.train.seg", - w_dict=None): - """ - create train data - """ - return data_reader(train_dir, w_dict, True) - - -def scdb_test_data(test_file, w_dict): - """ - test_set=["car", "lbs", "spot", "weibo", - "baby", "toutiao", "3c", "movie", "haogan"] - """ - return data_reader(test_file, w_dict) - - -def bow_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2): - """ - bow net - """ - emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim]) - bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') - bow_tanh = fluid.layers.tanh(bow) - fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") - fc_2 = fluid.layers.fc(input=fc_1, size=hid_dim2, act="tanh") - prediction = fluid.layers.fc(input=[fc_2], size=class_dim, act="softmax") - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def cnn_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - win_size=3): - """ - conv net - """ - emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim]) - - conv_3 = fluid.nets.sequence_conv_pool( - input=emb, - num_filters=hid_dim, - filter_size=win_size, - act="tanh", - pool_type="max") - - fc_1 = fluid.layers.fc(input=[conv_3], size=hid_dim2) - - prediction = fluid.layers.fc(input=[fc_1], size=class_dim, act="softmax") - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def lstm_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): - """ - lstm net - """ - emb = fluid.layers.embedding( - input=data, - size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) - - fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - - lstm_h, c = fluid.layers.dynamic_lstm( - input=fc0, size=hid_dim * 4, is_reverse=False) - - lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max') - lstm_max_tanh = fluid.layers.tanh(lstm_max) - - fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh') - - prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax') - - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def bilstm_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): - """ - lstm net - """ - emb = fluid.layers.embedding( - input=data, - size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) - - fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - - rfc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - - lstm_h, c = fluid.layers.dynamic_lstm( - input=fc0, size=hid_dim * 4, is_reverse=False) - - rlstm_h, c = fluid.layers.dynamic_lstm( - input=rfc0, size=hid_dim * 4, is_reverse=True) - - lstm_last = fluid.layers.sequence_last_step(input=lstm_h) - rlstm_last = fluid.layers.sequence_last_step(input=rlstm_h) - - lstm_last_tanh = fluid.layers.tanh(lstm_last) - rlstm_last_tanh = fluid.layers.tanh(rlstm_last) - - lstm_concat = fluid.layers.concat(input=[lstm_last, rlstm_last], axis=1) - - fc1 = fluid.layers.fc(input=lstm_concat, size=hid_dim2, act='tanh') - - prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax') - - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def gru_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): - """ - gru net - """ - emb = fluid.layers.embedding( - input=data, - size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) - - fc0 = fluid.layers.fc(input=emb, size=hid_dim * 3) - - gru_h = fluid.layers.dynamic_gru(input=fc0, size=hid_dim, is_reverse=False) - - gru_max = fluid.layers.sequence_pool(input=gru_h, pool_type='max') - gru_max_tanh = fluid.layers.tanh(gru_max) - - fc1 = fluid.layers.fc(input=gru_max_tanh, size=hid_dim2, act='tanh') - - prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax') - - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def infer(test_reader, use_cuda, model_path=None): - """ - inference function - """ - if model_path is None: - print(str(model_path) + " cannot be found") - return - - place = fluid.CPUPlace() - exe = fluid.Executor(place) - - inference_scope = fluid.Scope() - with fluid.scope_guard(inference_scope): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(model_path, exe) - - class2_list, class3_list = [], [] - for each_test_reader in test_reader: - class2_acc, class3_acc = 0.0, 0.0 - total_count, neu_count = 0, 0 - - for data in each_test_reader(): - pred = exe.run(inference_program, - feed=data2pred(data, place), - fetch_list=fetch_targets, - return_numpy=True) - - for i, val in enumerate(data): - pos_score = pred[0][i, 1] - true_label = val[1] - if true_label == 2.0 and pos_score > 0.5: - class2_acc += 1 - if true_label == 0.0 and pos_score < 0.5: - class2_acc += 1 - - if true_label == 2.0 and pos_score > 0.55: - class3_acc += 1 - if true_label == 1.0 and pos_score > 0.45 and pos_score <= 0.55: - class3_acc += 1 - if true_label == 0.0 and pos_score <= 0.45: - class3_acc += 1 - - if true_label == 1.0: - neu_count += 1 - - total_count += len(data) - - class2_acc = class2_acc / (total_count - neu_count) - class3_acc = class3_acc / total_count - class2_list.append(class2_acc) - class3_list.append(class3_acc) - - class2_acc = sum(class2_list) / len(class2_list) - class3_acc = sum(class3_list) / len(class3_list) - print("[test info] model_path: %s, class2_acc: %f, class3_acc: %f" % - (model_path, class2_acc, class3_acc)) - - -def start_train(train_reader, - test_reader, - word_dict, - network, - use_cuda, - parallel, - save_dirname, - lr=0.2, - batch_size=128, - pass_num=30): - """ - train network - """ - data = fluid.layers.data( - name="words", shape=[1], dtype="int64", lod_level=1) - - label = fluid.layers.data(name="label", shape=[1], dtype="int64") - - cost, acc, pred = network(data, label, len(word_dict) + 1) - - sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=lr) - sgd_optimizer.minimize(cost) - - place = fluid.CPUPlace() - feeder = fluid.DataFeeder(feed_list=[data, label], place=place) - - start_exe = fluid.Executor(place) - start_exe.run(fluid.default_startup_program()) - - exe = fluid.ParallelExecutor(use_cuda, loss_name=cost.name) - for pass_id in six.moves.xrange(pass_num): - total_acc, total_cost, total_count, avg_cost, avg_acc = 0.0, 0.0, 0.0, 0.0, 0.0 - for data in train_reader(): - cost_val, acc_val = exe.run(feed=feeder.feed(data), - fetch_list=[cost.name, acc.name]) - cost_val_list, acc_val_list = np.array(cost_val), np.array(acc_val) - total_cost += cost_val_list.sum() * len(data) - total_acc += acc_val_list.sum() * len(data) - total_count += len(data) - - avg_cost = total_cost / total_count - avg_acc = total_acc / total_count - print("[train info]: pass_id: %d, avg_acc: %f, avg_cost: %f" % - (pass_id, avg_acc, avg_cost)) - - gpu_place = fluid.CUDAPlace(0) - save_exe = fluid.Executor(gpu_place) - epoch_model = save_dirname + "/" + "epoch" + str(pass_id) - fluid.io.save_inference_model(epoch_model, ["words"], pred, save_exe) - infer(test_reader, False, epoch_model) - - -def train_net(vocab="./thirdparty/train.vocab", - train_dir="./train", - test_list=["car", "spot", "weibo", "lbs"]): - """ - w_dict = scdb_word_dict(vocab=vocab) - test_files = [ "./thirdparty" + os.sep + f for f in test_list] - - train_reader = paddle.batch( - scdb_train_data(train_dir, w_dict), - batch_size = 256) - - test_reader = [paddle.batch(scdb_test_data(test_file, w_dict), batch_size = 50) \ - for test_file in test_files] - """ - w_dict = paddle.dataset.imdb.word_dict() - print("dict ready") - train_reader = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.imdb.train(w_dict), buf_size=50000), - batch_size=128) - - test_reader = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.imdb.test(w_dict), buf_size=50000), - batch_size=128) - test_reader = [test_reader] - start_train( - train_reader, - test_reader, - w_dict, - bilstm_net, - use_cuda=True, - parallel=False, - save_dirname="scdb_bilstm_model", - lr=0.05, - pass_num=10, - batch_size=256) - - -if __name__ == "__main__": - train_net() diff --git a/PaddleNLP/unarchived/text_classification/clouds/scdb_single_card.py b/PaddleNLP/unarchived/text_classification/clouds/scdb_single_card.py deleted file mode 100644 index c75369cae0a8e52ab6fa3014517f321fdc2d163c..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/clouds/scdb_single_card.py +++ /dev/null @@ -1,486 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest -import contextlib -import paddle -import paddle.fluid as fluid -import numpy as np -import six -import sys -import time -import os -import json -import random - - -def to_lodtensor(data, place): - """ - convert to LODtensor - """ - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for l in seq_lens: - cur_len += l - lod.append(cur_len) - flattened_data = np.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - res = fluid.LoDTensor() - res.set(flattened_data, place) - res.set_lod([lod]) - return res - - -def load_vocab(filename): - """ - load imdb vocabulary - """ - vocab = {} - with open(filename) as f: - wid = 0 - for line in f: - vocab[line.strip()] = wid - wid += 1 - vocab[""] = len(vocab) - return vocab - - -def data2tensor(data, place): - """ - data2tensor - """ - input_seq = to_lodtensor([x[0] for x in data], place) - y_data = np.array([x[1] for x in data]).astype("int64") - y_data = y_data.reshape([-1, 1]) - return {"words": input_seq, "label": y_data} - - -def data2pred(data, place): - """ - data2tensor - """ - input_seq = to_lodtensor([x[0] for x in data], place) - y_data = np.array([x[1] for x in data]).astype("int64") - y_data = y_data.reshape([-1, 1]) - return {"words": input_seq} - - -def load_dict(vocab): - """ - Load dict from vocab - """ - word_dict = dict() - with open(vocab, "r") as fin: - for line in fin: - cols = line.strip("\r\n").decode("gb18030").split("\t") - word_dict[cols[0]] = int(cols[1]) - return word_dict - - -def save_dict(word_dict, vocab): - """ - Save dict into file - """ - with open(vocab, "w") as fout: - for k, v in six.iteritems(word_dict): - outstr = ("%s\t%s\n" % (k, v)).encode("gb18030") - fout.write(outstr) - - -def build_dict(fname): - """ - build word dict using trainset - """ - word_dict = dict() - with open(fname, "r") as fin: - for line in fin: - try: - words = line.strip("\r\n").decode("gb18030").split("\t")[ - 1].split(" ") - except: - sys.stderr.write("[warning] build_dict: decode error\n") - continue - for w in words: - if w not in word_dict: - word_dict[w] = len(word_dict) - return word_dict - - -def scdb_word_dict(vocab="scdb_data/train_set/train.vocab"): - """ - get word_dict - """ - if not os.path.exists(vocab): - w_dict = build_dict(train_file) - save_dict(w_dict, vocab) - else: - w_dict = load_dict(vocab) - w_dict[""] = len(w_dict) - return w_dict - - -def data_reader(fname, word_dict, is_dir=False): - """ - Convert word sequence into slot - """ - unk_id = len(word_dict) - all_data = [] - filelist = [] - if is_dir: - filelist = [fname + os.sep + f for f in os.listdir(fname)] - else: - filelist = [fname] - - for each_name in filelist: - with open(each_name, "r") as fin: - for line in fin: - try: - cols = line.strip("\r\n").decode("gb18030").split("\t") - except: - sys.stderr.write("warning: ignore decode error\n") - continue - - label = int(cols[0]) - wids = [ - word_dict[x] if x in word_dict else unk_id - for x in cols[1].split(" ") - ] - all_data.append((wids, label)) - - random.shuffle(all_data) - - def reader(): - for doc, label in all_data: - yield doc, label - - return reader - - -def scdb_train_data(train_dir="scdb_data/train_set/corpus.train.seg", - w_dict=None): - """ - create train data - """ - return data_reader(train_dir, w_dict, True) - - -def scdb_test_data(test_file, w_dict): - """ - test_set=["car", "lbs", "spot", "weibo", - "baby", "toutiao", "3c", "movie", "haogan"] - """ - return data_reader(test_file, w_dict) - - -def bow_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2): - """ - bow net - """ - emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim]) - bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') - bow_tanh = fluid.layers.tanh(bow) - fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") - fc_2 = fluid.layers.fc(input=fc_1, size=hid_dim2, act="tanh") - prediction = fluid.layers.fc(input=[fc_2], size=class_dim, act="softmax") - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def cnn_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - win_size=3): - """ - conv net - """ - emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim]) - - conv_3 = fluid.nets.sequence_conv_pool( - input=emb, - num_filters=hid_dim, - filter_size=win_size, - act="tanh", - pool_type="max") - - fc_1 = fluid.layers.fc(input=[conv_3], size=hid_dim2) - - prediction = fluid.layers.fc(input=[fc_1], size=class_dim, act="softmax") - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def lstm_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): - """ - lstm net - """ - emb = fluid.layers.embedding( - input=data, - size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) - - fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - - lstm_h, c = fluid.layers.dynamic_lstm( - input=fc0, size=hid_dim * 4, is_reverse=False) - - lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max') - lstm_max_tanh = fluid.layers.tanh(lstm_max) - - fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh') - - prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax') - - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def bilstm_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): - """ - lstm net - """ - emb = fluid.layers.embedding( - input=data, - size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) - - fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - - rfc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - - lstm_h, c = fluid.layers.dynamic_lstm( - input=fc0, size=hid_dim * 4, is_reverse=False) - - rlstm_h, c = fluid.layers.dynamic_lstm( - input=rfc0, size=hid_dim * 4, is_reverse=True) - - lstm_last = fluid.layers.sequence_last_step(input=lstm_h) - rlstm_last = fluid.layers.sequence_last_step(input=rlstm_h) - - lstm_last_tanh = fluid.layers.tanh(lstm_last) - rlstm_last_tanh = fluid.layers.tanh(rlstm_last) - - lstm_concat = fluid.layers.concat(input=[lstm_last, rlstm_last], axis=1) - - fc1 = fluid.layers.fc(input=lstm_concat, size=hid_dim2, act='tanh') - - prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax') - - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def gru_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): - """ - gru net - """ - emb = fluid.layers.embedding( - input=data, - size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) - - fc0 = fluid.layers.fc(input=emb, size=hid_dim * 3) - - gru_h = fluid.layers.dynamic_gru(input=fc0, size=hid_dim, is_reverse=False) - - gru_max = fluid.layers.sequence_pool(input=gru_h, pool_type='max') - gru_max_tanh = fluid.layers.tanh(gru_max) - - fc1 = fluid.layers.fc(input=gru_max_tanh, size=hid_dim2, act='tanh') - - prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax') - - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def infer(test_reader, use_cuda, model_path=None): - """ - inference function - """ - if model_path is None: - print(str(model_path) + " cannot be found") - return - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - - inference_scope = fluid.Scope() - with fluid.scope_guard(inference_scope): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(model_path, exe) - - class2_list, class3_list = [], [] - for each_test_reader in test_reader: - class2_acc, class3_acc = 0.0, 0.0 - total_count, neu_count = 0, 0 - - for data in each_test_reader(): - pred = exe.run(inference_program, - feed=data2pred(data, place), - fetch_list=fetch_targets, - return_numpy=True) - - for i, val in enumerate(data): - pos_score = pred[0][i, 1] - true_label = val[1] - if true_label == 2.0 and pos_score > 0.5: - class2_acc += 1 - if true_label == 0.0 and pos_score < 0.5: - class2_acc += 1 - - if true_label == 2.0 and pos_score > 0.55: - class3_acc += 1 - if true_label == 1.0 and pos_score > 0.45 and pos_score <= 0.55: - class3_acc += 1 - if true_label == 0.0 and pos_score <= 0.45: - class3_acc += 1 - - if true_label == 1.0: - neu_count += 1 - - total_count += len(data) - - class2_acc = class2_acc / (total_count - neu_count) - class3_acc = class3_acc / total_count - class2_list.append(class2_acc) - class3_list.append(class3_acc) - - class2_acc = sum(class2_list) / len(class2_list) - class3_acc = sum(class3_list) / len(class3_list) - print("[test info] model_path: %s, class2_acc: %f, class3_acc: %f" % - (model_path, class2_acc, class3_acc)) - - -def start_train(train_reader, - test_reader, - word_dict, - network, - use_cuda, - parallel, - save_dirname, - lr=0.2, - batch_size=128, - pass_num=30): - """ - train network - """ - data = fluid.layers.data( - name="words", shape=[1], dtype="int64", lod_level=1) - - label = fluid.layers.data(name="label", shape=[1], dtype="int64") - - cost, acc, pred = network(data, label, len(word_dict) + 1) - - sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=lr) - sgd_optimizer.minimize(cost) - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - feeder = fluid.DataFeeder(feed_list=[data, label], place=place) - - exe.run(fluid.default_startup_program()) - for pass_id in six.moves.xrange(pass_num): - data_size, data_count, total_acc, total_cost = 0, 0, 0.0, 0.0 - for data in train_reader(): - avg_cost_np, avg_acc_np = exe.run(fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[cost, acc]) - data_size = len(data) - total_acc += data_size * avg_acc_np - total_cost += data_size * avg_cost_np - data_count += data_size - - avg_cost = total_cost / data_count - avg_acc = total_acc / data_count - print("[train info]: pass_id: %d, avg_acc: %f, avg_cost: %f" % - (pass_id, avg_acc, avg_cost)) - - epoch_model = save_dirname + "/" + "epoch" + str(pass_id) - fluid.io.save_inference_model(epoch_model, ["words"], pred, exe) - infer(test_reader, False, epoch_model) - - -def train_net(vocab="./thirdparty/train.vocab", - train_dir="./train", - test_list=["car", "spot", "weibo", "lbs"]): - w_dict = scdb_word_dict(vocab=vocab) - test_files = ["./thirdparty" + os.sep + f for f in test_list] - - train_reader = paddle.batch( - scdb_train_data(train_dir, w_dict), batch_size=256) - - test_reader = [paddle.batch(scdb_test_data(test_file, w_dict), batch_size = 50) \ - for test_file in test_files] - - start_train( - train_reader, - test_reader, - w_dict, - bow_net, - use_cuda=False, - parallel=False, - save_dirname="scdb_bow_model", - lr=0.002, - pass_num=10, - batch_size=256) - - -if __name__ == "__main__": - train_net() diff --git a/PaddleNLP/unarchived/text_classification/infer.py b/PaddleNLP/unarchived/text_classification/infer.py deleted file mode 100644 index 9990e1f443cdc1efd46f7a7f6225e110761f6c36..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/infer.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import time -import unittest -import contextlib -import numpy as np - -import paddle -import paddle.fluid as fluid - -import utils - - -def infer(test_reader, use_cuda, model_path=None): - """ - inference function - """ - if model_path is None: - print(str(model_path) + " cannot be found") - return - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - - inference_scope = fluid.Scope() - with fluid.scope_guard(inference_scope): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(model_path, exe) - - total_acc = 0.0 - total_count = 0 - for data in test_reader(): - acc = exe.run(inference_program, - feed=utils.data2tensor(data, place), - fetch_list=fetch_targets, - return_numpy=True) - total_acc += acc[0] * len(data) - total_count += len(data) - - avg_acc = total_acc / total_count - print("model_path: %s, avg_acc: %f" % (model_path, avg_acc)) - - -if __name__ == "__main__": - word_dict, train_reader, test_reader = utils.prepare_data( - "imdb", self_dict=False, batch_size=128, buf_size=50000) - - model_path = sys.argv[1] - for i in range(30): - epoch_path = model_path + "/" + "epoch" + str(i) - infer(test_reader, use_cuda=False, model_path=epoch_path) diff --git a/PaddleNLP/unarchived/text_classification/nets.py b/PaddleNLP/unarchived/text_classification/nets.py deleted file mode 100644 index 4dc83aa432786ca393dde1f5e3039d3c83d5ff33..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/nets.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import time -import numpy as np - -import paddle -import paddle.fluid as fluid - - -def bow_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2): - """ - bow net - """ - emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim]) - bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') - bow_tanh = fluid.layers.tanh(bow) - fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") - fc_2 = fluid.layers.fc(input=fc_1, size=hid_dim2, act="tanh") - prediction = fluid.layers.fc(input=[fc_2], size=class_dim, act="softmax") - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def cnn_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - win_size=3): - """ - conv net - """ - emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim]) - - conv_3 = fluid.nets.sequence_conv_pool( - input=emb, - num_filters=hid_dim, - filter_size=win_size, - act="tanh", - pool_type="max") - - fc_1 = fluid.layers.fc(input=[conv_3], size=hid_dim2) - - prediction = fluid.layers.fc(input=[fc_1], size=class_dim, act="softmax") - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def lstm_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): - """ - lstm net - """ - emb = fluid.layers.embedding( - input=data, - size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) - - fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - - lstm_h, c = fluid.layers.dynamic_lstm( - input=fc0, size=hid_dim * 4, is_reverse=False) - - lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max') - lstm_max_tanh = fluid.layers.tanh(lstm_max) - - fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh') - - prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax') - - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction - - -def gru_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): - """ - gru net - """ - emb = fluid.layers.embedding( - input=data, - size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) - - fc0 = fluid.layers.fc(input=emb, size=hid_dim * 3) - gru_h = fluid.layers.dynamic_gru(input=fc0, size=hid_dim, is_reverse=False) - gru_max = fluid.layers.sequence_pool(input=gru_h, pool_type='max') - gru_max_tanh = fluid.layers.tanh(gru_max) - fc1 = fluid.layers.fc(input=gru_max_tanh, size=hid_dim2, act='tanh') - prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax') - - cost = fluid.layers.cross_entropy(input=prediction, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc = fluid.layers.accuracy(input=prediction, label=label) - - return avg_cost, acc, prediction diff --git a/PaddleNLP/unarchived/text_classification/train.py b/PaddleNLP/unarchived/text_classification/train.py deleted file mode 100644 index 363e8e704f3aaa2bca70abaa04b308da5207a08f..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/train.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import six -import sys -import time -import unittest -import contextlib - -import paddle -import paddle.fluid as fluid - -import utils -from nets import bow_net -from nets import cnn_net -from nets import lstm_net -from nets import gru_net - - -def train(train_reader, - word_dict, - network, - use_cuda, - parallel, - save_dirname, - lr=0.2, - pass_num=30): - """ - train network - """ - data = fluid.layers.data( - name="words", shape=[1], dtype="int64", lod_level=1) - - label = fluid.layers.data(name="label", shape=[1], dtype="int64") - - if not parallel: - cost, acc, prediction = network(data, label, len(word_dict)) - else: - places = fluid.layers.device.get_places(device_count=2) - pd = fluid.layers.ParallelDo(places) - with pd.do(): - cost, acc, prediction = network( - pd.read_input(data), pd.read_input(label), len(word_dict)) - - pd.write_output(cost) - pd.write_output(acc) - - cost, acc = pd() - cost = fluid.layers.mean(cost) - acc = fluid.layers.mean(acc) - - sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=lr) - sgd_optimizer.minimize(cost) - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - feeder = fluid.DataFeeder(feed_list=[data, label], place=place) - - # For internal continuous evaluation - if "CE_MODE_X" in os.environ: - fluid.default_startup_program().random_seed = 110 - exe.run(fluid.default_startup_program()) - for pass_id in six.moves.xrange(pass_num): - pass_start = time.time() - data_size, data_count, total_acc, total_cost = 0, 0, 0.0, 0.0 - for data in train_reader(): - avg_cost_np, avg_acc_np = exe.run(fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[cost, acc]) - data_size = len(data) - total_acc += data_size * avg_acc_np - total_cost += data_size * avg_cost_np - data_count += data_size - avg_cost = total_cost / data_count - - avg_acc = total_acc / data_count - print("pass_id: %d, avg_acc: %f, avg_cost: %f, pass_time_cost: %f" % - (pass_id, avg_acc, avg_cost, time.time() - pass_start)) - - epoch_model = save_dirname + "/" + "epoch" + str(pass_id) - fluid.io.save_inference_model(epoch_model, ["words", "label"], acc, exe) - - pass_end = time.time() - # For internal continuous evaluation - if "CE_MODE_X" in os.environ: - print("kpis train_acc %f" % avg_acc) - print("kpis train_cost %f" % avg_cost) - print("kpis train_duration %f" % (pass_end - pass_start)) - - -def train_net(): - word_dict, train_reader, test_reader = utils.prepare_data( - "imdb", self_dict=False, batch_size=128, buf_size=50000) - - if sys.argv[1] == "bow": - train( - train_reader, - word_dict, - bow_net, - use_cuda=False, - parallel=False, - save_dirname="bow_model", - lr=0.002, - pass_num=30) - elif sys.argv[1] == "cnn": - train( - train_reader, - word_dict, - cnn_net, - use_cuda=True, - parallel=False, - save_dirname="cnn_model", - lr=0.01, - pass_num=30) - elif sys.argv[1] == "lstm": - train( - train_reader, - word_dict, - lstm_net, - use_cuda=True, - parallel=False, - save_dirname="lstm_model", - lr=0.05, - pass_num=30) - elif sys.argv[1] == "gru": - train( - train_reader, - word_dict, - gru_net, - use_cuda=True, - parallel=False, - save_dirname="gru_model", - lr=0.05, - pass_num=30) - else: - print("network name cannot be found!") - sys.exit(1) - - -if __name__ == "__main__": - train_net() diff --git a/PaddleNLP/unarchived/text_classification/utils.py b/PaddleNLP/unarchived/text_classification/utils.py deleted file mode 100644 index ecddc3cf41606064ef65f0bd0725e592337e38e5..0000000000000000000000000000000000000000 --- a/PaddleNLP/unarchived/text_classification/utils.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import sys -import time -import numpy as np - -import paddle -import paddle.fluid as fluid - - -def to_lodtensor(data, place): - """ - convert to LODtensor - """ - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for l in seq_lens: - cur_len += l - lod.append(cur_len) - flattened_data = np.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - res = fluid.LoDTensor() - res.set(flattened_data, place) - res.set_lod([lod]) - return res - - -def load_vocab(filename): - """ - load imdb vocabulary - """ - vocab = {} - with open(filename) as f: - wid = 0 - for line in f: - vocab[line.strip()] = wid - wid += 1 - vocab[""] = len(vocab) - return vocab - - -def data2tensor(data, place): - """ - data2tensor - """ - input_seq = to_lodtensor([x[0] for x in data], place) - y_data = np.array([x[1] for x in data]).astype("int64") - y_data = y_data.reshape([-1, 1]) - return {"words": input_seq, "label": y_data} - - -def prepare_data(data_type="imdb", - self_dict=False, - batch_size=128, - buf_size=50000): - """ - prepare data - """ - if self_dict: - word_dict = load_vocab(data_type + ".vocab") - else: - if data_type == "imdb": - word_dict = paddle.dataset.imdb.word_dict() - else: - raise RuntimeError("No such dataset") - - if data_type == "imdb": - if "CE_MODE_X" in os.environ: - train_reader = paddle.batch( - paddle.dataset.imdb.train(word_dict), batch_size=batch_size) - - test_reader = paddle.batch( - paddle.dataset.imdb.test(word_dict), batch_size=batch_size) - else: - train_reader = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.imdb.train(word_dict), buf_size=buf_size), - batch_size=batch_size) - - test_reader = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.imdb.test(word_dict), buf_size=buf_size), - batch_size=batch_size) - else: - raise RuntimeError("no such dataset") - - return word_dict, train_reader, test_reader diff --git a/PaddleNLP/unarchived/text_matching_on_quora/__init__.py b/PaddleNLP/unarchived/text_matching_on_quora/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/PaddleNLP/unarchived/text_matching_on_quora/.run_ce.sh b/PaddleRec/text_matching_on_quora/.run_ce.sh similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/.run_ce.sh rename to PaddleRec/text_matching_on_quora/.run_ce.sh diff --git a/PaddleNLP/unarchived/text_matching_on_quora/README.md b/PaddleRec/text_matching_on_quora/README.md similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/README.md rename to PaddleRec/text_matching_on_quora/README.md diff --git a/PaddleNLP/unarchived/chinese_ner/__init__.py b/PaddleRec/text_matching_on_quora/__init__.py similarity index 100% rename from PaddleNLP/unarchived/chinese_ner/__init__.py rename to PaddleRec/text_matching_on_quora/__init__.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/_ce.py b/PaddleRec/text_matching_on_quora/_ce.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/_ce.py rename to PaddleRec/text_matching_on_quora/_ce.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/cdssm_base.log b/PaddleRec/text_matching_on_quora/cdssm_base.log similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/cdssm_base.log rename to PaddleRec/text_matching_on_quora/cdssm_base.log diff --git a/PaddleNLP/unarchived/text_matching_on_quora/configs/__init__.py b/PaddleRec/text_matching_on_quora/configs/__init__.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/configs/__init__.py rename to PaddleRec/text_matching_on_quora/configs/__init__.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/configs/basic_config.py b/PaddleRec/text_matching_on_quora/configs/basic_config.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/configs/basic_config.py rename to PaddleRec/text_matching_on_quora/configs/basic_config.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/configs/cdssm.py b/PaddleRec/text_matching_on_quora/configs/cdssm.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/configs/cdssm.py rename to PaddleRec/text_matching_on_quora/configs/cdssm.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/configs/dec_att.py b/PaddleRec/text_matching_on_quora/configs/dec_att.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/configs/dec_att.py rename to PaddleRec/text_matching_on_quora/configs/dec_att.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/configs/infer_sent.py b/PaddleRec/text_matching_on_quora/configs/infer_sent.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/configs/infer_sent.py rename to PaddleRec/text_matching_on_quora/configs/infer_sent.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/configs/sse.py b/PaddleRec/text_matching_on_quora/configs/sse.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/configs/sse.py rename to PaddleRec/text_matching_on_quora/configs/sse.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/data/prepare_quora_data.sh b/PaddleRec/text_matching_on_quora/data/prepare_quora_data.sh similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/data/prepare_quora_data.sh rename to PaddleRec/text_matching_on_quora/data/prepare_quora_data.sh diff --git a/PaddleNLP/unarchived/text_matching_on_quora/imgs/README.md b/PaddleRec/text_matching_on_quora/imgs/README.md similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/imgs/README.md rename to PaddleRec/text_matching_on_quora/imgs/README.md diff --git a/PaddleNLP/unarchived/text_matching_on_quora/imgs/models_test_acc.png b/PaddleRec/text_matching_on_quora/imgs/models_test_acc.png similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/imgs/models_test_acc.png rename to PaddleRec/text_matching_on_quora/imgs/models_test_acc.png diff --git a/PaddleNLP/unarchived/text_matching_on_quora/metric.py b/PaddleRec/text_matching_on_quora/metric.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/metric.py rename to PaddleRec/text_matching_on_quora/metric.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/models/__init__.py b/PaddleRec/text_matching_on_quora/models/__init__.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/models/__init__.py rename to PaddleRec/text_matching_on_quora/models/__init__.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/models/cdssm.py b/PaddleRec/text_matching_on_quora/models/cdssm.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/models/cdssm.py rename to PaddleRec/text_matching_on_quora/models/cdssm.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/models/dec_att.py b/PaddleRec/text_matching_on_quora/models/dec_att.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/models/dec_att.py rename to PaddleRec/text_matching_on_quora/models/dec_att.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/models/infer_sent.py b/PaddleRec/text_matching_on_quora/models/infer_sent.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/models/infer_sent.py rename to PaddleRec/text_matching_on_quora/models/infer_sent.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/models/match_layers.py b/PaddleRec/text_matching_on_quora/models/match_layers.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/models/match_layers.py rename to PaddleRec/text_matching_on_quora/models/match_layers.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/models/my_layers.py b/PaddleRec/text_matching_on_quora/models/my_layers.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/models/my_layers.py rename to PaddleRec/text_matching_on_quora/models/my_layers.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/models/pwim.py b/PaddleRec/text_matching_on_quora/models/pwim.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/models/pwim.py rename to PaddleRec/text_matching_on_quora/models/pwim.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/models/sse.py b/PaddleRec/text_matching_on_quora/models/sse.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/models/sse.py rename to PaddleRec/text_matching_on_quora/models/sse.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/models/test.py b/PaddleRec/text_matching_on_quora/models/test.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/models/test.py rename to PaddleRec/text_matching_on_quora/models/test.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/pretrained_word2vec.py b/PaddleRec/text_matching_on_quora/pretrained_word2vec.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/pretrained_word2vec.py rename to PaddleRec/text_matching_on_quora/pretrained_word2vec.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/quora_question_pairs.py b/PaddleRec/text_matching_on_quora/quora_question_pairs.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/quora_question_pairs.py rename to PaddleRec/text_matching_on_quora/quora_question_pairs.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/train_and_evaluate.py b/PaddleRec/text_matching_on_quora/train_and_evaluate.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/train_and_evaluate.py rename to PaddleRec/text_matching_on_quora/train_and_evaluate.py diff --git a/PaddleNLP/unarchived/text_matching_on_quora/utils.py b/PaddleRec/text_matching_on_quora/utils.py similarity index 100% rename from PaddleNLP/unarchived/text_matching_on_quora/utils.py rename to PaddleRec/text_matching_on_quora/utils.py diff --git a/legacy/PaddleRL/DeepQNetwork/DQN_agent.py b/legacy/PaddleRL/DeepQNetwork/DQN_agent.py deleted file mode 100644 index 60f86a59bf63e9e4fc4edbea0159e4d8aa3a5b9f..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/DQN_agent.py +++ /dev/null @@ -1,190 +0,0 @@ -#-*- coding: utf-8 -*- - -import math -import numpy as np -import paddle.fluid as fluid -from paddle.fluid.param_attr import ParamAttr -from tqdm import tqdm - - -class DQNModel(object): - def __init__(self, state_dim, action_dim, gamma, hist_len, use_cuda=False): - self.img_height = state_dim[0] - self.img_width = state_dim[1] - self.action_dim = action_dim - self.gamma = gamma - self.exploration = 1.1 - self.update_target_steps = 10000 // 4 - self.hist_len = hist_len - self.use_cuda = use_cuda - - self.global_step = 0 - self._build_net() - - def _get_inputs(self): - return fluid.layers.data( - name='state', - shape=[self.hist_len, self.img_height, self.img_width], - dtype='float32'), \ - fluid.layers.data( - name='action', shape=[1], dtype='int32'), \ - fluid.layers.data( - name='reward', shape=[], dtype='float32'), \ - fluid.layers.data( - name='next_s', - shape=[self.hist_len, self.img_height, self.img_width], - dtype='float32'), \ - fluid.layers.data( - name='isOver', shape=[], dtype='bool') - - def _build_net(self): - self.predict_program = fluid.Program() - self.train_program = fluid.Program() - self._sync_program = fluid.Program() - - with fluid.program_guard(self.predict_program): - state, action, reward, next_s, isOver = self._get_inputs() - self.pred_value = self.get_DQN_prediction(state) - - with fluid.program_guard(self.train_program): - state, action, reward, next_s, isOver = self._get_inputs() - pred_value = self.get_DQN_prediction(state) - - reward = fluid.layers.clip(reward, min=-1.0, max=1.0) - - action_onehot = fluid.layers.one_hot(action, self.action_dim) - action_onehot = fluid.layers.cast(action_onehot, dtype='float32') - - pred_action_value = fluid.layers.reduce_sum( - fluid.layers.elementwise_mul(action_onehot, pred_value), dim=1) - - targetQ_predict_value = self.get_DQN_prediction(next_s, target=True) - best_v = fluid.layers.reduce_max(targetQ_predict_value, dim=1) - best_v.stop_gradient = True - - target = reward + (1.0 - fluid.layers.cast( - isOver, dtype='float32')) * self.gamma * best_v - cost = fluid.layers.square_error_cost(pred_action_value, target) - cost = fluid.layers.reduce_mean(cost) - - optimizer = fluid.optimizer.Adam(1e-3 * 0.5, epsilon=1e-3) - optimizer.minimize(cost) - - vars = list(self.train_program.list_vars()) - target_vars = list( - filter(lambda x: 'GRAD' not in x.name and 'target' in x.name, vars)) - - policy_vars_name = [ - x.name.replace('target', 'policy') for x in target_vars - ] - policy_vars = list(filter(lambda x: x.name in policy_vars_name, vars)) - - policy_vars.sort(key=lambda x: x.name) - target_vars.sort(key=lambda x: x.name) - - with fluid.program_guard(self._sync_program): - sync_ops = [] - for i, var in enumerate(policy_vars): - sync_op = fluid.layers.assign(policy_vars[i], target_vars[i]) - sync_ops.append(sync_op) - - # fluid exe - place = fluid.CUDAPlace(0) if self.use_cuda else fluid.CPUPlace() - self.exe = fluid.Executor(place) - self.exe.run(fluid.default_startup_program()) - - def get_DQN_prediction(self, image, target=False): - image = image / 255.0 - - variable_field = 'target' if target else 'policy' - - conv1 = fluid.layers.conv2d( - input=image, - num_filters=32, - filter_size=5, - stride=1, - padding=2, - act='relu', - param_attr=ParamAttr(name='{}_conv1'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv1_b'.format(variable_field))) - max_pool1 = fluid.layers.pool2d( - input=conv1, pool_size=2, pool_stride=2, pool_type='max') - - conv2 = fluid.layers.conv2d( - input=max_pool1, - num_filters=32, - filter_size=5, - stride=1, - padding=2, - act='relu', - param_attr=ParamAttr(name='{}_conv2'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv2_b'.format(variable_field))) - max_pool2 = fluid.layers.pool2d( - input=conv2, pool_size=2, pool_stride=2, pool_type='max') - - conv3 = fluid.layers.conv2d( - input=max_pool2, - num_filters=64, - filter_size=4, - stride=1, - padding=1, - act='relu', - param_attr=ParamAttr(name='{}_conv3'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv3_b'.format(variable_field))) - max_pool3 = fluid.layers.pool2d( - input=conv3, pool_size=2, pool_stride=2, pool_type='max') - - conv4 = fluid.layers.conv2d( - input=max_pool3, - num_filters=64, - filter_size=3, - stride=1, - padding=1, - act='relu', - param_attr=ParamAttr(name='{}_conv4'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv4_b'.format(variable_field))) - - flatten = fluid.layers.flatten(conv4, axis=1) - - out = fluid.layers.fc( - input=flatten, - size=self.action_dim, - param_attr=ParamAttr(name='{}_fc1'.format(variable_field)), - bias_attr=ParamAttr(name='{}_fc1_b'.format(variable_field))) - return out - - def act(self, state, train_or_test): - sample = np.random.random() - if train_or_test == 'train' and sample < self.exploration: - act = np.random.randint(self.action_dim) - else: - if np.random.random() < 0.01: - act = np.random.randint(self.action_dim) - else: - state = np.expand_dims(state, axis=0) - pred_Q = self.exe.run(self.predict_program, - feed={'state': state.astype('float32')}, - fetch_list=[self.pred_value])[0] - pred_Q = np.squeeze(pred_Q, axis=0) - act = np.argmax(pred_Q) - if train_or_test == 'train': - self.exploration = max(0.1, self.exploration - 1e-6) - return act - - def train(self, state, action, reward, next_state, isOver): - if self.global_step % self.update_target_steps == 0: - self.sync_target_network() - self.global_step += 1 - - action = np.expand_dims(action, -1) - self.exe.run(self.train_program, - feed={ - 'state': state.astype('float32'), - 'action': action.astype('int32'), - 'reward': reward, - 'next_s': next_state.astype('float32'), - 'isOver': isOver - }) - - def sync_target_network(self): - self.exe.run(self._sync_program) diff --git a/legacy/PaddleRL/DeepQNetwork/DoubleDQN_agent.py b/legacy/PaddleRL/DeepQNetwork/DoubleDQN_agent.py deleted file mode 100644 index 87997cd8bcedf9e0c7e6bd6034c8d20f06d9e198..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/DoubleDQN_agent.py +++ /dev/null @@ -1,200 +0,0 @@ -#-*- coding: utf-8 -*- - -import math -import numpy as np -import paddle.fluid as fluid -from paddle.fluid.param_attr import ParamAttr -from tqdm import tqdm - - -class DoubleDQNModel(object): - def __init__(self, state_dim, action_dim, gamma, hist_len, use_cuda=False): - self.img_height = state_dim[0] - self.img_width = state_dim[1] - self.action_dim = action_dim - self.gamma = gamma - self.exploration = 1.1 - self.update_target_steps = 10000 // 4 - self.hist_len = hist_len - self.use_cuda = use_cuda - - self.global_step = 0 - self._build_net() - - def _get_inputs(self): - return fluid.layers.data( - name='state', - shape=[self.hist_len, self.img_height, self.img_width], - dtype='float32'), \ - fluid.layers.data( - name='action', shape=[1], dtype='int32'), \ - fluid.layers.data( - name='reward', shape=[], dtype='float32'), \ - fluid.layers.data( - name='next_s', - shape=[self.hist_len, self.img_height, self.img_width], - dtype='float32'), \ - fluid.layers.data( - name='isOver', shape=[], dtype='bool') - - def _build_net(self): - self.predict_program = fluid.Program() - self.train_program = fluid.Program() - self._sync_program = fluid.Program() - - with fluid.program_guard(self.predict_program): - state, action, reward, next_s, isOver = self._get_inputs() - self.pred_value = self.get_DQN_prediction(state) - - with fluid.program_guard(self.train_program): - state, action, reward, next_s, isOver = self._get_inputs() - pred_value = self.get_DQN_prediction(state) - - reward = fluid.layers.clip(reward, min=-1.0, max=1.0) - - action_onehot = fluid.layers.one_hot(action, self.action_dim) - action_onehot = fluid.layers.cast(action_onehot, dtype='float32') - - pred_action_value = fluid.layers.reduce_sum( - fluid.layers.elementwise_mul(action_onehot, pred_value), dim=1) - - targetQ_predict_value = self.get_DQN_prediction(next_s, target=True) - - next_s_predcit_value = self.get_DQN_prediction(next_s) - greedy_action = fluid.layers.argmax(next_s_predcit_value, axis=1) - greedy_action = fluid.layers.unsqueeze(greedy_action, axes=[1]) - - predict_onehot = fluid.layers.one_hot(greedy_action, - self.action_dim) - best_v = fluid.layers.reduce_sum( - fluid.layers.elementwise_mul(predict_onehot, - targetQ_predict_value), - dim=1) - best_v.stop_gradient = True - - target = reward + (1.0 - fluid.layers.cast( - isOver, dtype='float32')) * self.gamma * best_v - cost = fluid.layers.square_error_cost(pred_action_value, target) - cost = fluid.layers.reduce_mean(cost) - - optimizer = fluid.optimizer.Adam(1e-3 * 0.5, epsilon=1e-3) - optimizer.minimize(cost) - - vars = list(self.train_program.list_vars()) - target_vars = list( - filter(lambda x: 'GRAD' not in x.name and 'target' in x.name, vars)) - - policy_vars_name = [ - x.name.replace('target', 'policy') for x in target_vars - ] - policy_vars = list(filter(lambda x: x.name in policy_vars_name, vars)) - - policy_vars.sort(key=lambda x: x.name) - target_vars.sort(key=lambda x: x.name) - - with fluid.program_guard(self._sync_program): - sync_ops = [] - for i, var in enumerate(policy_vars): - sync_op = fluid.layers.assign(policy_vars[i], target_vars[i]) - sync_ops.append(sync_op) - - # fluid exe - place = fluid.CUDAPlace(0) if self.use_cuda else fluid.CPUPlace() - self.exe = fluid.Executor(place) - self.exe.run(fluid.default_startup_program()) - - def get_DQN_prediction(self, image, target=False): - image = image / 255.0 - - variable_field = 'target' if target else 'policy' - - conv1 = fluid.layers.conv2d( - input=image, - num_filters=32, - filter_size=5, - stride=1, - padding=2, - act='relu', - param_attr=ParamAttr(name='{}_conv1'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv1_b'.format(variable_field))) - max_pool1 = fluid.layers.pool2d( - input=conv1, pool_size=2, pool_stride=2, pool_type='max') - - conv2 = fluid.layers.conv2d( - input=max_pool1, - num_filters=32, - filter_size=5, - stride=1, - padding=2, - act='relu', - param_attr=ParamAttr(name='{}_conv2'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv2_b'.format(variable_field))) - max_pool2 = fluid.layers.pool2d( - input=conv2, pool_size=2, pool_stride=2, pool_type='max') - - conv3 = fluid.layers.conv2d( - input=max_pool2, - num_filters=64, - filter_size=4, - stride=1, - padding=1, - act='relu', - param_attr=ParamAttr(name='{}_conv3'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv3_b'.format(variable_field))) - max_pool3 = fluid.layers.pool2d( - input=conv3, pool_size=2, pool_stride=2, pool_type='max') - - conv4 = fluid.layers.conv2d( - input=max_pool3, - num_filters=64, - filter_size=3, - stride=1, - padding=1, - act='relu', - param_attr=ParamAttr(name='{}_conv4'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv4_b'.format(variable_field))) - - flatten = fluid.layers.flatten(conv4, axis=1) - - out = fluid.layers.fc( - input=flatten, - size=self.action_dim, - param_attr=ParamAttr(name='{}_fc1'.format(variable_field)), - bias_attr=ParamAttr(name='{}_fc1_b'.format(variable_field))) - return out - - def act(self, state, train_or_test): - sample = np.random.random() - if train_or_test == 'train' and sample < self.exploration: - act = np.random.randint(self.action_dim) - else: - if np.random.random() < 0.01: - act = np.random.randint(self.action_dim) - else: - state = np.expand_dims(state, axis=0) - pred_Q = self.exe.run(self.predict_program, - feed={'state': state.astype('float32')}, - fetch_list=[self.pred_value])[0] - pred_Q = np.squeeze(pred_Q, axis=0) - act = np.argmax(pred_Q) - if train_or_test == 'train': - self.exploration = max(0.1, self.exploration - 1e-6) - return act - - def train(self, state, action, reward, next_state, isOver): - if self.global_step % self.update_target_steps == 0: - self.sync_target_network() - self.global_step += 1 - - action = np.expand_dims(action, -1) - self.exe.run(self.train_program, - feed={ - 'state': state.astype('float32'), - 'action': action.astype('int32'), - 'reward': reward, - 'next_s': next_state.astype('float32'), - 'isOver': isOver - }) - - def sync_target_network(self): - self.exe.run(self._sync_program) diff --git a/legacy/PaddleRL/DeepQNetwork/DuelingDQN_agent.py b/legacy/PaddleRL/DeepQNetwork/DuelingDQN_agent.py deleted file mode 100644 index 5d63adc594c50a027e1bd7cf3867287690ddaa3a..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/DuelingDQN_agent.py +++ /dev/null @@ -1,200 +0,0 @@ -#-*- coding: utf-8 -*- - -import math -import numpy as np -import paddle.fluid as fluid -from paddle.fluid.param_attr import ParamAttr -from tqdm import tqdm - - -class DuelingDQNModel(object): - def __init__(self, state_dim, action_dim, gamma, hist_len, use_cuda=False): - self.img_height = state_dim[0] - self.img_width = state_dim[1] - self.action_dim = action_dim - self.gamma = gamma - self.exploration = 1.1 - self.update_target_steps = 10000 // 4 - self.hist_len = hist_len - self.use_cuda = use_cuda - - self.global_step = 0 - self._build_net() - - def _get_inputs(self): - return fluid.layers.data( - name='state', - shape=[self.hist_len, self.img_height, self.img_width], - dtype='float32'), \ - fluid.layers.data( - name='action', shape=[1], dtype='int32'), \ - fluid.layers.data( - name='reward', shape=[], dtype='float32'), \ - fluid.layers.data( - name='next_s', - shape=[self.hist_len, self.img_height, self.img_width], - dtype='float32'), \ - fluid.layers.data( - name='isOver', shape=[], dtype='bool') - - def _build_net(self): - self.predict_program = fluid.Program() - self.train_program = fluid.Program() - self._sync_program = fluid.Program() - - with fluid.program_guard(self.predict_program): - state, action, reward, next_s, isOver = self._get_inputs() - self.pred_value = self.get_DQN_prediction(state) - - with fluid.program_guard(self.train_program): - state, action, reward, next_s, isOver = self._get_inputs() - pred_value = self.get_DQN_prediction(state) - - reward = fluid.layers.clip(reward, min=-1.0, max=1.0) - - action_onehot = fluid.layers.one_hot(action, self.action_dim) - action_onehot = fluid.layers.cast(action_onehot, dtype='float32') - - pred_action_value = fluid.layers.reduce_sum( - fluid.layers.elementwise_mul(action_onehot, pred_value), dim=1) - - targetQ_predict_value = self.get_DQN_prediction(next_s, target=True) - best_v = fluid.layers.reduce_max(targetQ_predict_value, dim=1) - best_v.stop_gradient = True - - target = reward + (1.0 - fluid.layers.cast( - isOver, dtype='float32')) * self.gamma * best_v - cost = fluid.layers.square_error_cost(pred_action_value, target) - cost = fluid.layers.reduce_mean(cost) - - optimizer = fluid.optimizer.Adam(1e-3 * 0.5, epsilon=1e-3) - optimizer.minimize(cost) - - vars = list(self.train_program.list_vars()) - target_vars = list( - filter(lambda x: 'GRAD' not in x.name and 'target' in x.name, vars)) - - policy_vars_name = [ - x.name.replace('target', 'policy') for x in target_vars - ] - policy_vars = list(filter(lambda x: x.name in policy_vars_name, vars)) - - policy_vars.sort(key=lambda x: x.name) - target_vars.sort(key=lambda x: x.name) - - with fluid.program_guard(self._sync_program): - sync_ops = [] - for i, var in enumerate(policy_vars): - sync_op = fluid.layers.assign(policy_vars[i], target_vars[i]) - sync_ops.append(sync_op) - - # fluid exe - place = fluid.CUDAPlace(0) if self.use_cuda else fluid.CPUPlace() - self.exe = fluid.Executor(place) - self.exe.run(fluid.default_startup_program()) - - def get_DQN_prediction(self, image, target=False): - image = image / 255.0 - - variable_field = 'target' if target else 'policy' - - conv1 = fluid.layers.conv2d( - input=image, - num_filters=32, - filter_size=5, - stride=1, - padding=2, - act='relu', - param_attr=ParamAttr(name='{}_conv1'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv1_b'.format(variable_field))) - max_pool1 = fluid.layers.pool2d( - input=conv1, pool_size=2, pool_stride=2, pool_type='max') - - conv2 = fluid.layers.conv2d( - input=max_pool1, - num_filters=32, - filter_size=5, - stride=1, - padding=2, - act='relu', - param_attr=ParamAttr(name='{}_conv2'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv2_b'.format(variable_field))) - max_pool2 = fluid.layers.pool2d( - input=conv2, pool_size=2, pool_stride=2, pool_type='max') - - conv3 = fluid.layers.conv2d( - input=max_pool2, - num_filters=64, - filter_size=4, - stride=1, - padding=1, - act='relu', - param_attr=ParamAttr(name='{}_conv3'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv3_b'.format(variable_field))) - max_pool3 = fluid.layers.pool2d( - input=conv3, pool_size=2, pool_stride=2, pool_type='max') - - conv4 = fluid.layers.conv2d( - input=max_pool3, - num_filters=64, - filter_size=3, - stride=1, - padding=1, - act='relu', - param_attr=ParamAttr(name='{}_conv4'.format(variable_field)), - bias_attr=ParamAttr(name='{}_conv4_b'.format(variable_field))) - - flatten = fluid.layers.flatten(conv4, axis=1) - - value = fluid.layers.fc( - input=flatten, - size=1, - param_attr=ParamAttr(name='{}_value_fc'.format(variable_field)), - bias_attr=ParamAttr(name='{}_value_fc_b'.format(variable_field))) - - advantage = fluid.layers.fc( - input=flatten, - size=self.action_dim, - param_attr=ParamAttr(name='{}_advantage_fc'.format(variable_field)), - bias_attr=ParamAttr( - name='{}_advantage_fc_b'.format(variable_field))) - - Q = advantage + (value - fluid.layers.reduce_mean( - advantage, dim=1, keep_dim=True)) - return Q - - def act(self, state, train_or_test): - sample = np.random.random() - if train_or_test == 'train' and sample < self.exploration: - act = np.random.randint(self.action_dim) - else: - if np.random.random() < 0.01: - act = np.random.randint(self.action_dim) - else: - state = np.expand_dims(state, axis=0) - pred_Q = self.exe.run(self.predict_program, - feed={'state': state.astype('float32')}, - fetch_list=[self.pred_value])[0] - pred_Q = np.squeeze(pred_Q, axis=0) - act = np.argmax(pred_Q) - if train_or_test == 'train': - self.exploration = max(0.1, self.exploration - 1e-6) - return act - - def train(self, state, action, reward, next_state, isOver): - if self.global_step % self.update_target_steps == 0: - self.sync_target_network() - self.global_step += 1 - - action = np.expand_dims(action, -1) - self.exe.run(self.train_program, - feed={ - 'state': state.astype('float32'), - 'action': action.astype('int32'), - 'reward': reward, - 'next_s': next_state.astype('float32'), - 'isOver': isOver - }) - - def sync_target_network(self): - self.exe.run(self._sync_program) diff --git a/legacy/PaddleRL/DeepQNetwork/README.md b/legacy/PaddleRL/DeepQNetwork/README.md deleted file mode 100644 index 1edeaaa884318ec3a530ec4fdb7d031d07411b56..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/README.md +++ /dev/null @@ -1,67 +0,0 @@ -[中文版](README_cn.md) - -## Reproduce DQN, DoubleDQN, DuelingDQN model with Fluid version of PaddlePaddle -Based on PaddlePaddle's next-generation API Fluid, the DQN model of deep reinforcement learning is reproduced, and the same level of indicators of the paper is reproduced in the classic Atari game. The model receives the image of the game as input, and uses the end-to-end model to directly predict the next step. The repository contains the following three types of models: -+ DQN in -[Human-level Control Through Deep Reinforcement Learning](http://www.nature.com/nature/journal/v518/n7540/full/nature14236.html) -+ DoubleDQN in: -[Deep Reinforcement Learning with Double Q-Learning](https://www.aaai.org/ocs/index.php/AAAI/AAAI16/paper/viewPaper/12389) -+ DuelingDQN in: -[Dueling Network Architectures for Deep Reinforcement Learning](http://proceedings.mlr.press/v48/wangf16.html) - -## Atari benchmark & performance - -### Atari games introduction - -Please see [here](https://gym.openai.com/envs/#atari) to know more about Atari game. - -### Pong game result - -The average game rewards that can be obtained for the three models as the number of training steps changes during the training are as follows(about 3 hours/1 Million steps): - -
-DQN result -
- -## How to use -### Dependencies: -+ python2.7 -+ gym -+ tqdm -+ opencv-python -+ paddlepaddle-gpu>=1.0.0 -+ ale_python_interface - -### Install Dependencies: -+ Install PaddlePaddle: - recommended to compile and install PaddlePaddle from source code -+ Install other dependencies: - ``` - pip install -r requirement.txt - pip install gym[atari] - ``` - Install ale_python_interface, please see [here](https://github.com/mgbellemare/Arcade-Learning-Environment). - -### Start Training: -``` -# To train a model for Pong game with gpu (use DQN model as default) -python train.py --rom ./rom_files/pong.bin --use_cuda - -# To train a model for Pong with DoubleDQN -python train.py --rom ./rom_files/pong.bin --use_cuda --alg DoubleDQN - -# To train a model for Pong with DuelingDQN -python train.py --rom ./rom_files/pong.bin --use_cuda --alg DuelingDQN -``` - -To train more games, you can install more rom files from [here](https://github.com/openai/atari-py/tree/master/atari_py/atari_roms). - -### Start Testing: -``` -# Play the game with saved best model and calculate the average rewards -python play.py --rom ./rom_files/pong.bin --use_cuda --model_path ./saved_model/DQN-pong - -# Play the game with visualization -python play.py --rom ./rom_files/pong.bin --use_cuda --model_path ./saved_model/DQN-pong --viz 0.01 -``` -[Here](https://pan.baidu.com/s/1gIsbNw5V7tMeb74ojx-TMA) is saved models for Pong and Breakout games. You can use it to play the game directly. diff --git a/legacy/PaddleRL/DeepQNetwork/README_cn.md b/legacy/PaddleRL/DeepQNetwork/README_cn.md deleted file mode 100644 index 640d775ad8fed2be360d308b6c5df41c86d77c04..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/README_cn.md +++ /dev/null @@ -1,71 +0,0 @@ -## 基于PaddlePaddle的Fluid版本复现DQN, DoubleDQN, DuelingDQN三个模型 - -基于PaddlePaddle下一代API Fluid复现了深度强化学习领域的DQN模型，在经典的Atari 游戏上复现了论文同等水平的指标，模型接收游戏的图像作为输入，采用端到端的模型直接预测下一步要执行的控制信号，本仓库一共包含以下3类模型： -+ DQN模型： -[Human-level Control Through Deep Reinforcement Learning](http://www.nature.com/nature/journal/v518/n7540/full/nature14236.html) -+ DoubleDQN模型： -[Deep Reinforcement Learning with Double Q-Learning](https://www.aaai.org/ocs/index.php/AAAI/AAAI16/paper/viewPaper/12389) -+ DuelingDQN模型： -[Dueling Network Architectures for Deep Reinforcement Learning](http://proceedings.mlr.press/v48/wangf16.html) - -## 模型效果：Atari游戏表现 - -### Atari游戏介绍 - -请点击[这里](https://gym.openai.com/envs/#atari)了解Atari游戏。 - -### Pong游戏训练结果 -三个模型在训练过程中随着训练步数的变化，能得到的平均游戏奖励如下图所示（大概3小时每1百万步）： - -
-DQN result -
- -## 使用教程 - -### 依赖: -+ python2.7 -+ gym -+ tqdm -+ opencv-python -+ paddlepaddle-gpu>=1.0.0 -+ ale_python_interface - -### 下载依赖： - -+ 安装PaddlePaddle： - 建议通过PaddlePaddle源码进行编译安装 -+ 下载其它依赖： - ``` - pip install -r requirement.txt - pip install gym[atari] - ``` - 安装ale_python_interface可以参考[这里](https://github.com/mgbellemare/Arcade-Learning-Environment) - -### 训练模型： - -``` -# 使用GPU训练Pong游戏（默认使用DQN模型） -python train.py --rom ./rom_files/pong.bin --use_cuda - -# 训练DoubleDQN模型 -python train.py --rom ./rom_files/pong.bin --use_cuda --alg DoubleDQN - -# 训练DuelingDQN模型 -python train.py --rom ./rom_files/pong.bin --use_cuda --alg DuelingDQN -``` - -训练更多游戏，可以从[这里](https://github.com/openai/atari-py/tree/master/atari_py/atari_roms)下载游戏rom - -### 测试模型： - -``` -# Play the game with saved model and calculate the average rewards -# 使用训练过程中保存的最好模型玩游戏，以及计算平均奖励（rewards） -python play.py --rom ./rom_files/pong.bin --use_cuda --model_path ./saved_model/DQN-pong - -# 以可视化的形式来玩游戏 -python play.py --rom ./rom_files/pong.bin --use_cuda --model_path ./saved_model/DQN-pong --viz 0.01 -``` - -[这里](https://pan.baidu.com/s/1gIsbNw5V7tMeb74ojx-TMA)是Pong和Breakout游戏训练好的模型，可以直接用来测试。 diff --git a/legacy/PaddleRL/DeepQNetwork/assets/dqn.png b/legacy/PaddleRL/DeepQNetwork/assets/dqn.png deleted file mode 100644 index f8f8d12f9887cdab62f09b52597ec187a4c8107c..0000000000000000000000000000000000000000 Binary files a/legacy/PaddleRL/DeepQNetwork/assets/dqn.png and /dev/null differ diff --git a/legacy/PaddleRL/DeepQNetwork/atari.py b/legacy/PaddleRL/DeepQNetwork/atari.py deleted file mode 100644 index ec793cba15ddc1c42986689eaad5773875a4ffde..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/atari.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- - -import numpy as np -import os -import cv2 -import threading - -import gym -from gym import spaces -from gym.envs.atari.atari_env import ACTION_MEANING - -from atari_py import ALEInterface - -__all__ = ['AtariPlayer'] - -ROM_URL = "https://github.com/openai/atari-py/tree/master/atari_py/atari_roms" -_ALE_LOCK = threading.Lock() -""" -The following AtariPlayer are copied or modified from tensorpack/tensorpack: - https://github.com/tensorpack/tensorpack/blob/master/examples/DeepQNetwork/atari.py -""" - - -class AtariPlayer(gym.Env): - """ - A wrapper for ALE emulator, with configurations to mimic DeepMind DQN settings. - Info: - score: the accumulated reward in the current game - gameOver: True when the current game is Over - """ - - def __init__(self, - rom_file, - viz=0, - frame_skip=4, - nullop_start=30, - live_lost_as_eoe=True, - max_num_frames=0): - """ - Args: - rom_file: path to the rom - frame_skip: skip every k frames and repeat the action - viz: visualization to be done. - Set to 0 to disable. - Set to a positive number to be the delay between frames to show. - Set to a string to be a directory to store frames. - nullop_start: start with random number of null ops. - live_losts_as_eoe: consider lost of lives as end of episode. Useful for training. - max_num_frames: maximum number of frames per episode. - """ - super(AtariPlayer, self).__init__() - assert os.path.isfile(rom_file), \ - "rom {} not found. Please download at {}".format(rom_file, ROM_URL) - - try: - ALEInterface.setLoggerMode(ALEInterface.Logger.Error) - except AttributeError: - print("You're not using latest ALE") - - # avoid simulator bugs: https://github.com/mgbellemare/Arcade-Learning-Environment/issues/86 - with _ALE_LOCK: - self.ale = ALEInterface() - self.ale.setInt(b"random_seed", np.random.randint(0, 30000)) - self.ale.setInt(b"max_num_frames_per_episode", max_num_frames) - self.ale.setBool(b"showinfo", False) - - self.ale.setInt(b"frame_skip", 1) - self.ale.setBool(b'color_averaging', False) - # manual.pdf suggests otherwise. - self.ale.setFloat(b'repeat_action_probability', 0.0) - - # viz setup - if isinstance(viz, str): - assert os.path.isdir(viz), viz - self.ale.setString(b'record_screen_dir', viz) - viz = 0 - if isinstance(viz, int): - viz = float(viz) - self.viz = viz - if self.viz and isinstance(self.viz, float): - self.windowname = os.path.basename(rom_file) - cv2.startWindowThread() - cv2.namedWindow(self.windowname) - - self.ale.loadROM(rom_file.encode('utf-8')) - self.width, self.height = self.ale.getScreenDims() - self.actions = self.ale.getMinimalActionSet() - - self.live_lost_as_eoe = live_lost_as_eoe - self.frame_skip = frame_skip - self.nullop_start = nullop_start - - self.action_space = spaces.Discrete(len(self.actions)) - self.observation_space = spaces.Box(low=0, - high=255, - shape=(self.height, self.width), - dtype=np.uint8) - self._restart_episode() - - def get_action_meanings(self): - return [ACTION_MEANING[i] for i in self.actions] - - def _grab_raw_image(self): - """ - :returns: the current 3-channel image - """ - m = self.ale.getScreenRGB() - return m.reshape((self.height, self.width, 3)) - - def _current_state(self): - """ - returns: a gray-scale (h, w) uint8 image - """ - ret = self._grab_raw_image() - # avoid missing frame issue: max-pooled over the last screen - ret = np.maximum(ret, self.last_raw_screen) - if self.viz: - if isinstance(self.viz, float): - cv2.imshow(self.windowname, ret) - cv2.waitKey(int(self.viz * 1000)) - ret = ret.astype('float32') - # 0.299,0.587.0.114. same as rgb2y in torch/image - ret = cv2.cvtColor(ret, cv2.COLOR_RGB2GRAY) - return ret.astype('uint8') # to save some memory - - def _restart_episode(self): - with _ALE_LOCK: - self.ale.reset_game() - - # random null-ops start - n = np.random.randint(self.nullop_start) - self.last_raw_screen = self._grab_raw_image() - for k in range(n): - if k == n - 1: - self.last_raw_screen = self._grab_raw_image() - self.ale.act(0) - - def reset(self): - if self.ale.game_over(): - self._restart_episode() - return self._current_state() - - def step(self, act): - oldlives = self.ale.lives() - r = 0 - for k in range(self.frame_skip): - if k == self.frame_skip - 1: - self.last_raw_screen = self._grab_raw_image() - r += self.ale.act(self.actions[act]) - newlives = self.ale.lives() - if self.ale.game_over() or \ - (self.live_lost_as_eoe and newlives < oldlives): - break - - isOver = self.ale.game_over() - if self.live_lost_as_eoe: - isOver = isOver or newlives < oldlives - - info = {'ale.lives': newlives} - return self._current_state(), r, isOver, info diff --git a/legacy/PaddleRL/DeepQNetwork/atari_wrapper.py b/legacy/PaddleRL/DeepQNetwork/atari_wrapper.py deleted file mode 100644 index 81ec7e0ba0ee191f70591c16bfff560a62d3d395..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/atari_wrapper.py +++ /dev/null @@ -1,106 +0,0 @@ -# -*- coding: utf-8 -*- - -import numpy as np -from collections import deque - -import gym -from gym import spaces - -_v0, _v1 = gym.__version__.split('.')[:2] -assert int(_v0) > 0 or int(_v1) >= 10, gym.__version__ -""" -The following wrappers are copied or modified from openai/baselines: -https://github.com/openai/baselines/blob/master/baselines/common/atari_wrappers.py -""" - - -class MapState(gym.ObservationWrapper): - def __init__(self, env, map_func): - gym.ObservationWrapper.__init__(self, env) - self._func = map_func - - def observation(self, obs): - return self._func(obs) - - -class FrameStack(gym.Wrapper): - def __init__(self, env, k): - """Buffer observations and stack across channels (last axis).""" - gym.Wrapper.__init__(self, env) - self.k = k - self.frames = deque([], maxlen=k) - shp = env.observation_space.shape - chan = 1 if len(shp) == 2 else shp[2] - self.observation_space = spaces.Box(low=0, - high=255, - shape=(shp[0], shp[1], chan * k), - dtype=np.uint8) - - def reset(self): - """Clear buffer and re-fill by duplicating the first observation.""" - ob = self.env.reset() - for _ in range(self.k - 1): - self.frames.append(np.zeros_like(ob)) - self.frames.append(ob) - return self.observation() - - def step(self, action): - ob, reward, done, info = self.env.step(action) - self.frames.append(ob) - return self.observation(), reward, done, info - - def observation(self): - assert len(self.frames) == self.k - return np.stack(self.frames, axis=0) - - -class _FireResetEnv(gym.Wrapper): - def __init__(self, env): - """Take action on reset for environments that are fixed until firing.""" - gym.Wrapper.__init__(self, env) - assert env.unwrapped.get_action_meanings()[1] == 'FIRE' - assert len(env.unwrapped.get_action_meanings()) >= 3 - - def reset(self): - self.env.reset() - obs, _, done, _ = self.env.step(1) - if done: - self.env.reset() - obs, _, done, _ = self.env.step(2) - if done: - self.env.reset() - return obs - - def step(self, action): - return self.env.step(action) - - -def FireResetEnv(env): - if isinstance(env, gym.Wrapper): - baseenv = env.unwrapped - else: - baseenv = env - if 'FIRE' in baseenv.get_action_meanings(): - return _FireResetEnv(env) - return env - - -class LimitLength(gym.Wrapper): - def __init__(self, env, k): - gym.Wrapper.__init__(self, env) - self.k = k - - def reset(self): - # This assumes that reset() will really reset the env. - # If the underlying env tries to be smart about reset - # (e.g. end-of-life), the assumption doesn't hold. - ob = self.env.reset() - self.cnt = 0 - return ob - - def step(self, action): - ob, r, done, info = self.env.step(action) - self.cnt += 1 - if self.cnt == self.k: - done = True - return ob, r, done, info diff --git a/legacy/PaddleRL/DeepQNetwork/expreplay.py b/legacy/PaddleRL/DeepQNetwork/expreplay.py deleted file mode 100644 index 5f27ca7286b5db7ac963bc25236be416fad50eb0..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/expreplay.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding: utf-8 -*- - -import numpy as np -import copy -from collections import deque, namedtuple - -Experience = namedtuple('Experience', ['state', 'action', 'reward', 'isOver']) - - -class ReplayMemory(object): - def __init__(self, max_size, state_shape, context_len): - self.max_size = int(max_size) - self.state_shape = state_shape - self.context_len = int(context_len) - - self.state = np.zeros((self.max_size, ) + state_shape, dtype='uint8') - self.action = np.zeros((self.max_size, ), dtype='int32') - self.reward = np.zeros((self.max_size, ), dtype='float32') - self.isOver = np.zeros((self.max_size, ), dtype='bool') - - self._curr_size = 0 - self._curr_pos = 0 - self._context = deque(maxlen=context_len - 1) - - def append(self, exp): - """append a new experience into replay memory - """ - if self._curr_size < self.max_size: - self._assign(self._curr_pos, exp) - self._curr_size += 1 - else: - self._assign(self._curr_pos, exp) - self._curr_pos = (self._curr_pos + 1) % self.max_size - if exp.isOver: - self._context.clear() - else: - self._context.append(exp) - - def recent_state(self): - """ maintain recent state for training""" - lst = list(self._context) - states = [np.zeros(self.state_shape, dtype='uint8')] * \ - (self._context.maxlen - len(lst)) - states.extend([k.state for k in lst]) - return states - - def sample(self, idx): - """ return state, action, reward, isOver, - note that some frames in state may be generated from last episode, - they should be removed from state - """ - state = np.zeros( - (self.context_len + 1, ) + self.state_shape, dtype=np.uint8) - state_idx = np.arange(idx, idx + self.context_len + 1) % self._curr_size - - # confirm that no frame was generated from last episode - has_last_episode = False - for k in range(self.context_len - 2, -1, -1): - to_check_idx = state_idx[k] - if self.isOver[to_check_idx]: - has_last_episode = True - state_idx = state_idx[k + 1:] - state[k + 1:] = self.state[state_idx] - break - - if not has_last_episode: - state = self.state[state_idx] - - real_idx = (idx + self.context_len - 1) % self._curr_size - action = self.action[real_idx] - reward = self.reward[real_idx] - isOver = self.isOver[real_idx] - return state, reward, action, isOver - - def __len__(self): - return self._curr_size - - def _assign(self, pos, exp): - self.state[pos] = exp.state - self.reward[pos] = exp.reward - self.action[pos] = exp.action - self.isOver[pos] = exp.isOver - - def sample_batch(self, batch_size): - """sample a batch from replay memory for training - """ - batch_idx = np.random.randint( - self._curr_size - self.context_len - 1, size=batch_size) - batch_idx = (self._curr_pos + batch_idx) % self._curr_size - batch_exp = [self.sample(i) for i in batch_idx] - return self._process_batch(batch_exp) - - def _process_batch(self, batch_exp): - state = np.asarray([e[0] for e in batch_exp], dtype='uint8') - reward = np.asarray([e[1] for e in batch_exp], dtype='float32') - action = np.asarray([e[2] for e in batch_exp], dtype='int8') - isOver = np.asarray([e[3] for e in batch_exp], dtype='bool') - return [state, action, reward, isOver] diff --git a/legacy/PaddleRL/DeepQNetwork/play.py b/legacy/PaddleRL/DeepQNetwork/play.py deleted file mode 100644 index 2c93da509d7cccb81d713c7aefd45a11ee28e8fb..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/play.py +++ /dev/null @@ -1,65 +0,0 @@ -#-*- coding: utf-8 -*- - -import argparse -import os -import numpy as np -import paddle.fluid as fluid - -from train import get_player -from tqdm import tqdm - - -def predict_action(exe, state, predict_program, feed_names, fetch_targets, - action_dim): - if np.random.random() < 0.01: - act = np.random.randint(action_dim) - else: - state = np.expand_dims(state, axis=0) - pred_Q = exe.run(predict_program, - feed={feed_names[0]: state.astype('float32')}, - fetch_list=fetch_targets)[0] - pred_Q = np.squeeze(pred_Q, axis=0) - act = np.argmax(pred_Q) - return act - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - '--use_cuda', action='store_true', help='if set, use cuda') - parser.add_argument('--rom', type=str, required=True, help='atari rom') - parser.add_argument( - '--model_path', type=str, required=True, help='dirname to load model') - parser.add_argument( - '--viz', - type=float, - default=0, - help='''viz: visualization setting: - Set to 0 to disable; - Set to a positive number to be the delay between frames to show. - ''') - args = parser.parse_args() - - env = get_player(args.rom, viz=args.viz) - - place = fluid.CUDAPlace(0) if args.use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - inference_scope = fluid.Scope() - with fluid.scope_guard(inference_scope): - [predict_program, feed_names, - fetch_targets] = fluid.io.load_inference_model(args.model_path, exe) - - episode_reward = [] - for _ in tqdm(xrange(30), desc='eval agent'): - state = env.reset() - total_reward = 0 - while True: - action = predict_action(exe, state, predict_program, feed_names, - fetch_targets, env.action_space.n) - state, reward, isOver, info = env.step(action) - total_reward += reward - if isOver: - break - episode_reward.append(total_reward) - eval_reward = np.mean(episode_reward) - print('Average reward of 30 epidose: {}'.format(eval_reward)) diff --git a/legacy/PaddleRL/DeepQNetwork/requirement.txt b/legacy/PaddleRL/DeepQNetwork/requirement.txt deleted file mode 100644 index 689eb324e6bd65aabbe44ca041ff7b3ddacb1943..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/requirement.txt +++ /dev/null @@ -1,5 +0,0 @@ -numpy -gym -tqdm -opencv-python -paddlepaddle-gpu>=1.0.0 diff --git a/legacy/PaddleRL/DeepQNetwork/rom_files/breakout.bin b/legacy/PaddleRL/DeepQNetwork/rom_files/breakout.bin deleted file mode 100644 index abab5a8c0a1890461a11b78d4265f1b794327793..0000000000000000000000000000000000000000 Binary files a/legacy/PaddleRL/DeepQNetwork/rom_files/breakout.bin and /dev/null differ diff --git a/legacy/PaddleRL/DeepQNetwork/rom_files/pong.bin b/legacy/PaddleRL/DeepQNetwork/rom_files/pong.bin deleted file mode 100644 index 14a5bdfc72548613c059938bdf712efdbb5d3806..0000000000000000000000000000000000000000 Binary files a/legacy/PaddleRL/DeepQNetwork/rom_files/pong.bin and /dev/null differ diff --git a/legacy/PaddleRL/DeepQNetwork/train.py b/legacy/PaddleRL/DeepQNetwork/train.py deleted file mode 100644 index dd7986d704aec0c0948f81ca7ddd69bbbd3ea239..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/DeepQNetwork/train.py +++ /dev/null @@ -1,181 +0,0 @@ -#-*- coding: utf-8 -*- - -from DQN_agent import DQNModel -from DoubleDQN_agent import DoubleDQNModel -from DuelingDQN_agent import DuelingDQNModel -from atari import AtariPlayer -import paddle.fluid as fluid -import gym -import argparse -import cv2 -from tqdm import tqdm -from expreplay import ReplayMemory, Experience -import numpy as np -import os - -from datetime import datetime -from atari_wrapper import FrameStack, MapState, FireResetEnv, LimitLength -from collections import deque - -UPDATE_FREQ = 4 - -MEMORY_SIZE = 1e6 -MEMORY_WARMUP_SIZE = MEMORY_SIZE // 20 -IMAGE_SIZE = (84, 84) -CONTEXT_LEN = 4 -ACTION_REPEAT = 4 # aka FRAME_SKIP -UPDATE_FREQ = 4 - - -def run_train_episode(agent, env, exp): - total_reward = 0 - state = env.reset() - step = 0 - while True: - step += 1 - context = exp.recent_state() - context.append(state) - context = np.stack(context, axis=0) - action = agent.act(context, train_or_test='train') - next_state, reward, isOver, _ = env.step(action) - exp.append(Experience(state, action, reward, isOver)) - # train model - # start training - if len(exp) > MEMORY_WARMUP_SIZE: - if step % UPDATE_FREQ == 0: - batch_all_state, batch_action, batch_reward, batch_isOver = exp.sample_batch( - args.batch_size) - batch_state = batch_all_state[:, :CONTEXT_LEN, :, :] - batch_next_state = batch_all_state[:, 1:, :, :] - agent.train(batch_state, batch_action, batch_reward, - batch_next_state, batch_isOver) - total_reward += reward - state = next_state - if isOver: - break - return total_reward, step - - -def get_player(rom, viz=False, train=False): - env = AtariPlayer( - rom, - frame_skip=ACTION_REPEAT, - viz=viz, - live_lost_as_eoe=train, - max_num_frames=60000) - env = FireResetEnv(env) - env = MapState(env, lambda im: cv2.resize(im, IMAGE_SIZE)) - if not train: - # in training, context is taken care of in expreplay buffer - env = FrameStack(env, CONTEXT_LEN) - return env - - -def eval_agent(agent, env): - episode_reward = [] - for _ in tqdm(range(30), desc='eval agent'): - state = env.reset() - total_reward = 0 - step = 0 - while True: - step += 1 - action = agent.act(state, train_or_test='test') - state, reward, isOver, info = env.step(action) - total_reward += reward - if isOver: - break - episode_reward.append(total_reward) - eval_reward = np.mean(episode_reward) - return eval_reward - - -def train_agent(): - env = get_player(args.rom, train=True) - test_env = get_player(args.rom) - exp = ReplayMemory(args.mem_size, IMAGE_SIZE, CONTEXT_LEN) - action_dim = env.action_space.n - - if args.alg == 'DQN': - agent = DQNModel(IMAGE_SIZE, action_dim, args.gamma, CONTEXT_LEN, - args.use_cuda) - elif args.alg == 'DoubleDQN': - agent = DoubleDQNModel(IMAGE_SIZE, action_dim, args.gamma, CONTEXT_LEN, - args.use_cuda) - elif args.alg == 'DuelingDQN': - agent = DuelingDQNModel(IMAGE_SIZE, action_dim, args.gamma, CONTEXT_LEN, - args.use_cuda) - else: - print('Input algorithm name error!') - return - - with tqdm(total=MEMORY_WARMUP_SIZE, desc='Memory warmup') as pbar: - while len(exp) < MEMORY_WARMUP_SIZE: - total_reward, step = run_train_episode(agent, env, exp) - pbar.update(step) - - # train - test_flag = 0 - save_flag = 0 - pbar = tqdm(total=1e8) - recent_100_reward = [] - total_step = 0 - max_reward = None - save_path = os.path.join(args.model_dirname, '{}-{}'.format( - args.alg, os.path.basename(args.rom).split('.')[0])) - while True: - # start epoch - total_reward, step = run_train_episode(agent, env, exp) - total_step += step - pbar.set_description('[train]exploration:{}'.format(agent.exploration)) - pbar.update(step) - - if total_step // args.test_every_steps == test_flag: - pbar.write("testing") - eval_reward = eval_agent(agent, test_env) - test_flag += 1 - print("eval_agent done, (steps, eval_reward): ({}, {})".format( - total_step, eval_reward)) - - if max_reward is None or eval_reward > max_reward: - max_reward = eval_reward - fluid.io.save_inference_model(save_path, ['state'], - agent.pred_value, agent.exe, - agent.predict_program) - pbar.close() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - '--alg', - type=str, - default='DQN', - help='Reinforcement learning algorithm, support: DQN, DoubleDQN, DuelingDQN' - ) - parser.add_argument( - '--use_cuda', action='store_true', help='if set, use cuda') - parser.add_argument( - '--gamma', - type=float, - default=0.99, - help='discount factor for accumulated reward computation') - parser.add_argument( - '--mem_size', - type=int, - default=1000000, - help='memory size for experience replay') - parser.add_argument( - '--batch_size', type=int, default=64, help='batch size for training') - parser.add_argument('--rom', help='atari rom', required=True) - parser.add_argument( - '--model_dirname', - type=str, - default='saved_model', - help='dirname to save model') - parser.add_argument( - '--test_every_steps', - type=int, - default=100000, - help='every steps number to run test') - args = parser.parse_args() - train_agent() diff --git a/legacy/PaddleRL/README.md b/legacy/PaddleRL/README.md deleted file mode 100644 index 5b8d2caf78d426a14b96f7d842eb88ed37bab233..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/README.md +++ /dev/null @@ -1,11 +0,0 @@ -PaddleRL -============ - -强化学习 --------- - -强化学习是近年来一个愈发重要的机器学习方向，特别是与深度学习相结合而形成的深度强化学习(Deep Reinforcement Learning, DRL)，取得了很多令人惊异的成就。人们所熟知的战胜人类顶级围棋职业选手的 AlphaGo 就是 DRL 应用的一个典型例子，除游戏领域外，其它的应用还包括机器人、自然语言处理等。 - -深度强化学习的开山之作是在Atari视频游戏中的成功应用， 其可直接接受视频帧这种高维输入并根据图像内容端到端地预测下一步的动作，所用到的模型被称为深度Q网络(Deep Q-Network, DQN)。本实例就是利用PaddlePaddle Fluid这个灵活的框架，实现了 DQN 及其变体，并测试了它们在 Atari 游戏中的表现。 - -- [DeepQNetwork](https://github.com/PaddlePaddle/models/blob/develop/PaddleRL/DeepQNetwork/README_cn.md) diff --git a/legacy/PaddleRL/policy_gradient/README.md b/legacy/PaddleRL/policy_gradient/README.md deleted file mode 100644 index b813aa124466597adfb80261bee7c2de22b95e67..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/policy_gradient/README.md +++ /dev/null @@ -1,171 +0,0 @@ -运行本目录下的程序示例需要使用PaddlePaddle的最新develop分枝。如果您的PaddlePaddle安装版本低于此要求，请按照[安装文档](http://www.paddlepaddle.org/docs/develop/documentation/zh/build_and_install/pip_install_cn.html)中的说明更新PaddlePaddle安装版本。 - ---- - -# Policy Gradient RL by PaddlePaddle -本文介绍了如何使用PaddlePaddle通过policy-based的强化学习方法来训练一个player（actor model）, 我们希望这个player可以完成简单的走阶梯任务。 - - 内容分为: - - - 任务描述 - - 模型 - - 策略（目标函数） - - 算法（Gradient ascent） - - PaddlePaddle实现 - - -## 1. 任务描述 -假设有一个阶梯，连接A、B点，player从A点出发，每一步只能向前走一步或向后走一步，到达B点即为完成任务。我们希望训练一个聪明的player，它知道怎么最快的从A点到达B点。 -我们在命令行以下边的形式模拟任务： -``` -A - O - - - - - B -``` -一个‘-'代表一个阶梯，A点在行头，B点在行末，O代表player当前在的位置。 - -## 2. Policy Gradient -### 2.1 模型 -#### inputyer -模型的输入是player观察到的当前阶梯的状态$S$, 要包含阶梯的长度和player当前的位置信息。 -在命令行模拟的情况下，player的位置和阶梯长度连个变量足以表示当前的状态，但是我们为了便于将这个demo推广到更复杂的任务场景，我们这里用一个向量来表示游戏状态$S$. -向量$S$的长度为阶梯的长度，每一维代表一个阶梯，player所在的位置为1，其它位置为0. -下边是一个例子： -``` -S = [0, 1, 0, 0] // 阶梯长度为4，player在第二个阶梯上。 -``` -#### hidden layer -隐藏层采用两个全连接layer `FC_1`和`FC_2`, 其中`FC_1` 的size为10， `FC_2`的size为2. - -#### output layer -我们使用softmax将`FC_2`的output映射为所有可能的动作（前进或后退）的概率分布（Probability of taking the action）,即为一个二维向量`act_probs`, 其中，`act_probs[0]` 为后退的概率，`act_probs[1]`为前进的概率。 - -#### 模型表示 -我将我们的player模型(actor)形式化表示如下： -$$a = \pi_\theta(s)$$ -其中$\theta$表示模型的参数，$s$是输入状态。 - - -### 2.2 策略（目标函数） -我们怎么评估一个player(模型)的好坏呢？首先我们定义几个术语： -我们让$\pi_\theta(s)$来玩一局游戏，$s_t$表示第$t$时刻的状态，$a_t$表示在状态$s_t$做出的动作，$r_t$表示做过动作$a_t$后得到的奖赏。 -一局游戏的过程可以表示如下： -$$\tau = [s_1, a_1, r_1, s_2, a_2, r_2 ... s_T, a_T, r_T] \tag{1}$$ - -一局游戏的奖励表示如下： -$$R(\tau) = \sum_{t=1}^Tr_t$$ - -player玩一局游戏，可能会出现多种操作序列$\tau$ ,某个$\tau$出现的概率是依赖于player model的$\theta$, 记做： -$$P(\tau | \theta)$$ -那么，给定一个$\theta$(player model), 玩一局游戏，期望得到的奖励是： -$$\overline {R}_\theta = \sum_\tau R(\tau)\sum_\tau R(\tau) P(\tau|\theta)$$ -大多数情况，我们无法穷举出所有的$\tau$，所以我们就抽取N个$\tau$来计算近似的期望： -$$\overline {R}_\theta = \sum_\tau R(\tau) P(\tau|\theta) \approx \frac{1}{N} \sum_{n=1}^N R(\tau^n)$$ - -$\overline {R}_\theta$就是我们需要的目标函数，它表示了一个参数为$\theta$的player玩一局游戏得分的期望，这个期望越大，代表这个player能力越强。 -### 2.3 算法（Gradient ascent） -我们的目标函数是$\overline {R}_\theta$, 我们训练的任务就是, 我们训练的任务就是: -$$\theta^* = \arg\max_\theta \overline {R}_\theta$$ - -为了找到理想的$\theta$，我们使用Gradient ascent方法不断在$\overline {R}_\theta$的梯度方向更新$\theta$，可表示如下： -$$\theta' = \theta + \eta * \bigtriangledown \overline {R}_\theta$$ - -$$ \bigtriangledown \overline {R}_\theta = \sum_\tau R(\tau) \bigtriangledown P(\tau|\theta)\\ -= \sum_\tau R(\tau) P(\tau|\theta) \frac{\bigtriangledown P(\tau|\theta)}{P(\tau|\theta)} \\ -=\sum_\tau R(\tau) P(\tau|\theta) {\bigtriangledown \log P(\tau|\theta)} $$ - - -$$P(\tau|\theta) = P(s_1)P(a_1|s_1,\theta)P(s_2, r_1|s_1,a_1)P(a_2|s_2,\theta)P(s_3,r_2|s_2,a_2)...P(a_t|s_t,\theta)P(s_{t+1}, r_t|s_t,a_t)\\ -=P(s_1) \sum_{t=1}^T P(a_t|s_t,\theta)P(s_{t+1}, r_t|s_t,a_t)$$ - -$$\log P(\tau|\theta) = \log P(s_1) + \sum_{t=1}^T [\log P(a_t|s_t,\theta) + \log P(s_{t+1}, r_t|s_t,a_t)]$$ - -$$ \bigtriangledown \log P(\tau|\theta) = \sum_{t=1}^T \bigtriangledown \log P(a_t|s_t,\theta)$$ - -$$ \bigtriangledown \overline {R}_\theta = \sum_\tau R(\tau) P(\tau|\theta) {\bigtriangledown \log P(\tau|\theta)} \\ -\approx \frac{1}{N} \sum_{n=1}^N R(\tau^n) {\bigtriangledown \log P(\tau|\theta)} \\ -= \frac{1}{N} \sum_{n=1}^N R(\tau^n) {\sum_{t=1}^T \bigtriangledown \log P(a_t|s_t,\theta)} \\ -= \frac{1}{N} \sum_{n=1}^N \sum_{t=1}^T R(\tau^n) { \bigtriangledown \log P(a_t|s_t,\theta)} \tag{11}$$ - -#### 2.3.2 导数解释 - -在使用深度学习框架进行训练求解时，一般用梯度下降方法，所以我们把Gradient ascent转为Gradient -descent, 重写等式$(5)(6)$为： - -$$\theta^* = \arg\min_\theta (-\overline {R}_\theta \tag{13}$$ -$$\theta' = \theta - \eta * \bigtriangledown (-\overline {R}_\theta)) \tag{14}$$ - -根据上一节的推导，$ (-\bigtriangledown \overline {R}_\theta) $结果如下： - -$$ -\bigtriangledown \overline {R}_\theta -= \frac{1}{N} \sum_{n=1}^N \sum_{t=1}^T R(\tau^n) { \bigtriangledown -\log P(a_t|s_t,\theta)} \tag{15}$$ - -根据等式(14), 我们的player的模型可以设计为： - -

-
-图 1 -

- -用户的在一局游戏中的一次操作可以用元组$(s_t, a_t)$, 就是在状态$s_t$状态下做了动作$a_t$, 我们通过图(1)中的前向网络计算出来cross entropy cost为$−\log P(a_t|s_t,\theta)$， 恰好是等式(15)中我们需要微分的一项。 -图1是我们需要的player模型，我用这个网络的前向计算可以预测任何状态下该做什么动作。但是怎么去训练学习这个网络呢？在等式(15)中还有一项$R(\tau^n)$, 我做反向梯度传播的时候要加上这一项，所以我们需要在图1基础上再加上$R(\tau^n)$， 如 图2 所示： - -

-
-图 2 -

- -图2就是我们最终的网络结构。 - -#### 2.3.3 直观理解 -对于等式(15),我只看游戏中的一步操作，也就是这一项: $R(\tau^n) { \bigtriangledown -\log P(a_t|s_t,\theta)}$, 我们可以简单的认为我们训练的目的是让 $R(\tau^n) {[ -\log P(a_t|s_t,\theta)]}$尽可能的小，也就是$R(\tau^n) \log P(a_t|s_t,\theta)$尽可能的大。 - -- 如果我们当前游戏局的奖励$R(\tau^n)$为正，那么我们希望当前操作的出现的概率$P(a_t|s_t,\theta)$尽可能大。 -- 如果我们当前游戏局的奖励$R(\tau^n)$为负，那么我们希望当前操作的出现的概率$P(a_t|s_t,\theta)$尽可能小。 - -#### 2.3.4 一个问题 - -一人犯错，诛连九族。一人得道，鸡犬升天。如果一局游戏得到奖励，我们希望帮助获得奖励的每一次操作都被重视；否则，导致惩罚的操作都要被冷落一次。 -是不是很有道理的样子？但是，如果有些游戏场景只有奖励，没有惩罚，怎么办？也就是所有的$R(\tau^n)$都为正。 -针对不同的游戏场景，我们有不同的解决方案： - -1. 每局游戏得分不一样：将每局的得分减去一个bias，结果就有正有负了。 -2. 每局游戏得分一样：把完成一局的时间作为计分因素，并减去一个bias. - -我们在第一章描述的游戏场景，需要用第二种 ，player每次到达终点都会收到1分的奖励，我们可以按完成任务所用的步数来定义奖励R. -更进一步，我们认为一局游戏中每步动作对结局的贡献是不同的，有聪明的动作，也有愚蠢的操作。直观的理解，一般是靠前的动作是愚蠢的，靠后的动作是聪明的。既然有了这个价值观，那么我们拿到1分的奖励，就不能平均分给每个动作了。 -如图3所示，让所有动作按先后排队，从后往前衰减地给每个动作奖励，然后再每个动作的奖励再减去所有动作奖励的平均值： - -

-
-图 3 -

- -## 3. 训练效果 - -demo运行训练效果如下，经过1000轮尝试，我们的player就学会了如何有效的完成任务了： - -``` ----------O epoch: 0; steps: 42 ----------O epoch: 1; steps: 77 ----------O epoch: 2; steps: 82 ----------O epoch: 3; steps: 64 ----------O epoch: 4; steps: 79 ----------O epoch: 501; steps: 19 ----------O epoch: 1001; steps: 9 ----------O epoch: 1501; steps: 9 ----------O epoch: 2001; steps: 11 ----------O epoch: 2501; steps: 9 ----------O epoch: 3001; steps: 9 ----------O epoch: 3002; steps: 9 ----------O epoch: 3003; steps: 9 ----------O epoch: 3004; steps: 9 ----------O epoch: 3005; steps: 9 ----------O epoch: 3006; steps: 9 ----------O epoch: 3007; steps: 9 ----------O epoch: 3008; steps: 9 ----------O epoch: 3009; steps: 9 ----------O epoch: 3010; steps: 11 ----------O epoch: 3011; steps: 9 ----------O epoch: 3012; steps: 9 ----------O epoch: 3013; steps: 9 ----------O epoch: 3014; steps: 9 -``` diff --git a/legacy/PaddleRL/policy_gradient/brain.py b/legacy/PaddleRL/policy_gradient/brain.py deleted file mode 100644 index c7e55e7e3eeccd37cd7f2eb9adb0ea79c5d33c33..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/policy_gradient/brain.py +++ /dev/null @@ -1,94 +0,0 @@ -import numpy as np -import paddle.fluid as fluid -# reproducible -np.random.seed(1) - - -class PolicyGradient: - def __init__( - self, - n_actions, - n_features, - learning_rate=0.01, - reward_decay=0.95, - output_graph=False, ): - self.n_actions = n_actions - self.n_features = n_features - self.lr = learning_rate - self.gamma = reward_decay - - self.ep_obs, self.ep_as, self.ep_rs = [], [], [] - - self.place = fluid.CPUPlace() - self.exe = fluid.Executor(self.place) - - def build_net(self): - - obs = fluid.layers.data( - name='obs', shape=[self.n_features], dtype='float32') - acts = fluid.layers.data(name='acts', shape=[1], dtype='int64') - vt = fluid.layers.data(name='vt', shape=[1], dtype='float32') - # fc1 - fc1 = fluid.layers.fc(input=obs, size=10, act="tanh") # tanh activation - # fc2 - self.all_act_prob = fluid.layers.fc(input=fc1, - size=self.n_actions, - act="softmax") - self.inferece_program = fluid.default_main_program().clone() - # to maximize total reward (log_p * R) is to minimize -(log_p * R) - neg_log_prob = fluid.layers.cross_entropy( - input=self.all_act_prob, - label=acts) # this is negative log of chosen action - neg_log_prob_weight = fluid.layers.elementwise_mul(x=neg_log_prob, y=vt) - loss = fluid.layers.reduce_mean( - neg_log_prob_weight) # reward guided loss - - sgd_optimizer = fluid.optimizer.SGD(self.lr) - sgd_optimizer.minimize(loss) - self.exe.run(fluid.default_startup_program()) - - def choose_action(self, observation): - prob_weights = self.exe.run(self.inferece_program, - feed={"obs": observation[np.newaxis, :]}, - fetch_list=[self.all_act_prob]) - prob_weights = np.array(prob_weights[0]) - # select action w.r.t the actions prob - action = np.random.choice( - range(prob_weights.shape[1]), p=prob_weights.ravel()) - return action - - def store_transition(self, s, a, r): - self.ep_obs.append(s) - self.ep_as.append(a) - self.ep_rs.append(r) - - def learn(self): - # discount and normalize episode reward - discounted_ep_rs_norm = self._discount_and_norm_rewards() - tensor_obs = np.vstack(self.ep_obs).astype("float32") - tensor_as = np.array(self.ep_as).astype("int64") - tensor_as = tensor_as.reshape([tensor_as.shape[0], 1]) - tensor_vt = discounted_ep_rs_norm.astype("float32")[:, np.newaxis] - # train on episode - self.exe.run( - fluid.default_main_program(), - feed={ - "obs": tensor_obs, # shape=[None, n_obs] - "acts": tensor_as, # shape=[None, ] - "vt": tensor_vt # shape=[None, ] - }) - self.ep_obs, self.ep_as, self.ep_rs = [], [], [] # empty episode data - return discounted_ep_rs_norm - - def _discount_and_norm_rewards(self): - # discount episode rewards - discounted_ep_rs = np.zeros_like(self.ep_rs) - running_add = 0 - for t in reversed(range(0, len(self.ep_rs))): - running_add = running_add * self.gamma + self.ep_rs[t] - discounted_ep_rs[t] = running_add - - # normalize episode rewards - discounted_ep_rs -= np.mean(discounted_ep_rs) - discounted_ep_rs /= np.std(discounted_ep_rs) - return discounted_ep_rs diff --git a/legacy/PaddleRL/policy_gradient/env.py b/legacy/PaddleRL/policy_gradient/env.py deleted file mode 100644 index e2cd972dbc9a3943aceb9763b9dabcd50a1e6df1..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/policy_gradient/env.py +++ /dev/null @@ -1,56 +0,0 @@ -import time -import sys -import numpy as np - - -class Env(): - def __init__(self, stage_len, interval): - self.stage_len = stage_len - self.end = self.stage_len - 1 - self.position = 0 - self.interval = interval - self.step = 0 - self.epoch = -1 - self.render = False - - def reset(self): - self.end = self.stage_len - 1 - self.position = 0 - self.epoch += 1 - self.step = 0 - if self.render: - self.draw(True) - - def status(self): - s = np.zeros([self.stage_len]).astype("float32") - s[self.position] = 1 - return s - - def move(self, action): - self.step += 1 - reward = 0.0 - done = False - if action == 0: - self.position = max(0, self.position - 1) - else: - self.position = min(self.end, self.position + 1) - if self.render: - self.draw() - if self.position == self.end: - reward = 1.0 - done = True - return reward, done, self.status() - - def draw(self, new_line=False): - if new_line: - print "" - else: - print "\r", - for i in range(self.stage_len): - if i == self.position: - sys.stdout.write("O") - else: - sys.stdout.write("-") - sys.stdout.write(" epoch: %d; steps: %d" % (self.epoch, self.step)) - sys.stdout.flush() - time.sleep(self.interval) diff --git a/legacy/PaddleRL/policy_gradient/images/PG_1.svg b/legacy/PaddleRL/policy_gradient/images/PG_1.svg deleted file mode 100644 index e2352ff57ceb70bdba013c55c35eb1dc1cabe275..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/policy_gradient/images/PG_1.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - Produced by OmniGraffle 6.0.5 2017-12-01 08:39Z神经网络Layer 1x_2y_2y_0x_1x_nSoftmaxy_ma_2a_0a_m. . .. . .. . .. . .s_tθy_t = P(a_t | s_t, θ)-log(y_t) = -logP(a_t | s_t, θ)CROSS ENTROPY = diff --git a/legacy/PaddleRL/policy_gradient/images/PG_2.svg b/legacy/PaddleRL/policy_gradient/images/PG_2.svg deleted file mode 100644 index 3697bf9feca0861c9c0b2da29980ba4c86a3f4d7..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/policy_gradient/images/PG_2.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - Produced by OmniGraffle 6.0.5 2017-12-01 08:39Z神经网络 2Layer 1s_tYFCa_t-logP(a_t | s_t, θ)SoftmaxR(τ^n)Cross EntropyMul-R(τ^n)logP(a_t | s_t, θ)θ diff --git a/legacy/PaddleRL/policy_gradient/images/PG_3.svg b/legacy/PaddleRL/policy_gradient/images/PG_3.svg deleted file mode 100644 index 97b56c3fe1188e603a3bf5f6eabf7ea0ea3072c7..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/policy_gradient/images/PG_3.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - Produced by OmniGraffle 6.0.5 2017-12-01 09:42Z神经网络 3Layer 1Ra_2= 0.9 * … a_1a_(t-1)a_t= 0.9^2 *= 0.9^t * -= mean(a_1, a_2 … a_t) diff --git a/legacy/PaddleRL/policy_gradient/run.py b/legacy/PaddleRL/policy_gradient/run.py deleted file mode 100644 index 6f2f8c381a9d6452c5d7dfefb41f05eb4551d73a..0000000000000000000000000000000000000000 --- a/legacy/PaddleRL/policy_gradient/run.py +++ /dev/null @@ -1,29 +0,0 @@ -from brain import PolicyGradient -from env import Env -import numpy as np - -n_actions = 2 -interval = 0.01 -stage_len = 10 -epoches = 10000 - -if __name__ == "__main__": - - brain = PolicyGradient(n_actions, stage_len) - e = Env(stage_len, interval) - brain.build_net() - done = False - - for epoch in range(epoches): - if (epoch % 500 == 1) or epoch < 5 or epoch > 3000: - e.render = True - else: - e.render = False - e.reset() - while not done: - s = e.status() - action = brain.choose_action(s) - r, done, _ = e.move(action) - brain.store_transition(s, action, r) - done = False - brain.learn() diff --git a/legacy/README.cn.md b/legacy/README.cn.md deleted file mode 100644 index 72fb35ff3b239d8fa5e226f84aa09f084f593697..0000000000000000000000000000000000000000 --- a/legacy/README.cn.md +++ /dev/null @@ -1,136 +0,0 @@ -# models 简介 - -[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](https://github.com/PaddlePaddle/models) -[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](https://github.com/PaddlePaddle/models) -[![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE) - -PaddlePaddle提供了丰富的运算单元，帮助大家以模块化的方式构建起千变万化的深度学习模型来解决不同的应用问题。这里，我们针对常见的机器学习任务，提供了不同的神经网络模型供大家学习和使用。 - - -## 1. 词向量 - -词向量用一个实向量表示词语，向量的每个维都表示文本的某种潜在语法或语义特征，是深度学习应用于自然语言处理领域最成功的概念和成果之一。广义的，词向量也可以应用于普通离散特征。词向量的学习通常都是一个无监督的学习过程，因此，可以充分利用海量的无标记数据以捕获特征之间的关系，也可以有效地解决特征稀疏、标签数据缺失、数据噪声等问题。然而，在常见词向量学习方法中，模型最后一层往往会遇到一个超大规模的分类问题，是计算性能的瓶颈。 - -在词向量任务中，我们向大家展示如何使用Hierarchical-Sigmoid 和噪声对比估计（Noise Contrastive Estimation，NCE）来加速词向量的学习。 - -- 1.1 [Hsigmoid加速词向量训练](https://github.com/PaddlePaddle/models/tree/develop/hsigmoid) -- 1.2 [噪声对比估计加速词向量训练](https://github.com/PaddlePaddle/models/tree/develop/nce_cost) - - -## 2. RNN 语言模型 - -语言模型是自然语言处理领域里一个重要的基础模型，除了得到词向量（语言模型训练的副产物），还可以帮助我们生成文本。给定若干个词，语言模型可以帮助我们预测下一个最可能出现的词。 - -在利用语言模型生成文本的任务中，我们重点介绍循环神经网络语言模型，大家可以通过文档中的使用说明快速适配到自己的训练语料，完成自动写诗、自动写散文等有趣的模型。 - -- 2.1 [使用循环神经网络语言模型生成文本](https://github.com/PaddlePaddle/models/tree/develop/generate_sequence_by_rnn_lm) - -## 3. 点击率预估 - -点击率预估模型预判用户对一条广告点击的概率，对每次广告的点击情况做出预测，是广告技术的核心算法之一。逻谛斯克回归对大规模稀疏特征有着很好的学习能力，在点击率预估任务发展的早期一统天下。近年来，DNN 模型由于其强大的学习能力逐渐接过点击率预估任务的大旗。 - -在点击率预估任务中，我们首先给出谷歌提出的 Wide & Deep 模型。这一模型融合了适用于学习抽象特征的DNN和适用于大规模稀疏特征的逻谛斯克回归两者的优点，可以作为一种相对成熟的模型框架使用，在工业界也有一定的应用。同时，我们提供基于因子分解机的深度神经网络模型，该模型融合了因子分解机和深度神经网络，分别建模输入属性之间的低阶交互和高阶交互。 - -- 3.1 [Wide & deep 点击率预估模型](https://github.com/PaddlePaddle/models/tree/develop/ctr/README.cn.md) -- 3.2 [基于深度因子分解机的点击率预估模型](https://github.com/PaddlePaddle/models/tree/develop/deep_fm) - -## 4. 文本分类 - -文本分类是自然语言处理领域最基础的任务之一，深度学习方法能够免除复杂的特征工程，直接使用原始文本作为输入，数据驱动地最优化分类准确率。 - -在文本分类任务中，我们以情感分类任务为例，提供了基于DNN的非序列文本分类模型，以及基于CNN的序列模型供大家学习和使用（基于LSTM的模型见PaddleBook中[情感分类](http://www.paddlepaddle.org/docs/develop/book/06.understand_sentiment/index.cn.html)一课）。 - -- 4.1 [基于DNN/CNN的情感分类](https://github.com/PaddlePaddle/models/tree/develop/text_classification) -- 4.2 [基于双层序列的文本分类模型](https://github.com/PaddlePaddle/models/tree/develop/nested_sequence/text_classification) - -## 5. 排序学习 - -排序学习(Learning to Rank， LTR)是信息检索和搜索引擎研究的核心问题之一，通过机器学习方法学习一个分值函数对待排序的候选进行打分，再根据分值的高低确定序关系。深度神经网络可以用来建模分值函数，构成各类基于深度学习的LTR模型。 - -在排序学习任务中，我们介绍基于RankLoss损失函数Pairwise排序模型和基于LambdaRank损失函数的Listwise排序模型(Pointwise学习策略见PaddleBook中[推荐系统](http://www.paddlepaddle.org/docs/develop/book/05.recommender_system/index.cn.html)一课）。 - -- 5.1 [基于Pairwise和Listwise的排序学习](https://github.com/PaddlePaddle/models/tree/develop/ltr) - -## 6. 结构化语义模型 - -深度结构化语义模型是一种基于神经网络的语义匹配模型框架，可以用于学习两路信息实体或是文本之间的语义相似性。DSSM使用DNN、CNN或是RNN将两路信息实体或是文本映射到同一个连续的低纬度语义空间中。在这个语义空间中，两路实体或是文本可以同时进行表示，然后，通过定义距离度量和匹配函数来刻画并学习不同实体或是文本在同一个语义空间内的语义相似性。 - -在结构化语义模型任务中，我们演示如何建模两个字符串之间的语义相似度。模型支持DNN(全连接前馈网络)、CNN(卷积网络)、RNN(递归神经网络)等不同的网络结构，以及分类、回归、排序等不同损失函数。本例采用最简单的文本数据作为输入，通过替换自己的训练和预测数据，便可以在真实场景中使用。 - -- 6.1 [深度结构化语义模型](https://github.com/PaddlePaddle/models/tree/develop/dssm/README.cn.md) - -## 7. 命名实体识别 - -给定输入序列，序列标注模型为序列中每一个元素贴上一个类别标签，是自然语言处理领域最基础的任务之一。随着深度学习方法的不断发展，利用循环神经网络学习输入序列的特征表示，条件随机场（Conditional Random Field, CRF）在特征基础上完成序列标注任务，逐渐成为解决序列标注问题的标配解决方案。 - -在序列标注任务中，我们以命名实体识别（Named Entity Recognition，NER）任务为例，介绍如何训练一个端到端的序列标注模型。 - -- 7.1 [命名实体识别](https://github.com/PaddlePaddle/models/tree/develop/sequence_tagging_for_ner) - -## 8. 序列到序列学习 - -序列到序列学习实现两个甚至是多个不定长模型之间的映射，有着广泛的应用，包括：机器翻译、智能对话与问答、广告创意语料生成、自动编码（如金融画像编码）、判断多个文本串之间的语义相关性等。 - -在序列到序列学习任务中，我们首先以机器翻译任务为例，提供了多种改进模型供大家学习和使用。包括：不带注意力机制的序列到序列映射模型，这一模型是所有序列到序列学习模型的基础；使用Scheduled Sampling改善RNN模型在生成任务中的错误累积问题；带外部记忆机制的神经机器翻译，通过增强神经网络的记忆能力，来完成复杂的序列到序列学习任务。除机器翻译任务之外，我们也提供了一个基于深层LSTM网络生成古诗词，实现同语言生成的模型。 - -- 8.1 [无注意力机制的神经机器翻译](https://github.com/PaddlePaddle/models/tree/develop/nmt_without_attention/README.cn.md) -- 8.2 [使用Scheduled Sampling改善翻译质量](https://github.com/PaddlePaddle/models/tree/develop/scheduled_sampling) -- 8.3 [带外部记忆机制的神经机器翻译](https://github.com/PaddlePaddle/models/tree/develop/mt_with_external_memory) -- 8.4 [生成古诗词](https://github.com/PaddlePaddle/models/tree/develop/generate_chinese_poetry) - -## 9. 阅读理解 - -当深度学习以及各类新技术不断推动自然语言处理领域向前发展时，我们不禁会问：应该如何确认模型真正理解了人类特有的自然语言，具备一定的理解和推理能力？纵观NLP领域的各类经典问题：词法分析、句法分析、情感分类、写诗等，这些问题的经典解决方案，从技术原理上距离“语言理解”仍有一定距离。为了衡量现有NLP技术到“语言理解”这一终极目标之间的差距，我们需要一个有足够难度且可量化可复现的任务，这也是阅读理解问题提出的初衷。尽管目前的研究现状表明在现有阅读理解数据集上表现良好的模型，依然没有做到真正的语言理解，但机器阅读理解依然被视为是检验模型向理解语言迈进的一个重要任务。 - -阅读理解本质上也是自动问答的一种，模型“阅读”一段文字后回答给定的问题，在这一任务中，我们介绍使用Learning to Search 方法，将阅读理解转化为从段落中寻找答案所在句子，答案在句子中的起始位置，以及答案在句子中的结束位置，这样一个多步决策过程。 - -- 9.1 [Globally Normalized Reader](https://github.com/PaddlePaddle/models/tree/develop/globally_normalized_reader) - -## 10. 自动问答 - -自动问答（Question Answering）系统利用计算机自动回答用户提出的问题，是验证机器是否具备自然语言理解能力的重要任务之一，其研究历史可以追溯到人工智能的原点。与检索系统相比，自动问答系统是信息服务的一种高级形式，系统返回给用户的不再是排序后的基于关键字匹配的检索结果，而是精准的自然语言答案。 - -在自动问答任务中，我们介绍基于深度学习的端到端问答系统，将自动问答转化为一个序列标注问题。端对端问答系统试图通过从高质量的"问题-证据（Evidence）-答案"数据中学习，建立一个联合学习模型，同时学习语料库、知识库、问句语义表示之间的语义映射关系，将传统的问句语义解析、文本检索、答案抽取与生成的复杂步骤转变为一个可学习过程。 - -- 10.1 [基于序列标注的事实型自动问答模型](https://github.com/PaddlePaddle/models/tree/develop/neural_qa) - -## 11. 图像分类 - -图像相比文字能够提供更加生动、容易理解及更具艺术感的信息，是人们转递与交换信息的重要来源。图像分类是根据图像的语义信息对不同类别图像进行区分，是计算机视觉中重要的基础问题，也是图像检测、图像分割、物体跟踪、行为分析等其他高层视觉任务的基础，在许多领域都有着广泛的应用。如：安防领域的人脸识别和智能视频分析等，交通领域的交通场景识别，互联网领域基于内容的图像检索和相册自动归类，医学领域的图像识别等。 - -在图像分类任务中，我们向大家介绍如何训练AlexNet、VGG、GoogLeNet、ResNet、Inception-v4、Inception-Resnet-V2和Xception模型。同时提供了能够将Caffe或TensorFlow训练好的模型文件转换为PaddlePaddle模型文件的模型转换工具。 - -- 11.1 [将Caffe模型文件转换为PaddlePaddle模型文件](https://github.com/PaddlePaddle/models/tree/develop/image_classification/caffe2paddle) -- 11.2 [将TensorFlow模型文件转换为PaddlePaddle模型文件](https://github.com/PaddlePaddle/models/tree/develop/image_classification/tf2paddle) -- 11.3 [AlexNet](https://github.com/PaddlePaddle/models/tree/develop/image_classification) -- 11.4 [VGG](https://github.com/PaddlePaddle/models/tree/develop/image_classification) -- 11.5 [Residual Network](https://github.com/PaddlePaddle/models/tree/develop/image_classification) -- 11.6 [Inception-v4](https://github.com/PaddlePaddle/models/tree/develop/image_classification) -- 11.7 [Inception-Resnet-V2](https://github.com/PaddlePaddle/models/tree/develop/image_classification) -- 11.8 [Xception](https://github.com/PaddlePaddle/models/tree/develop/image_classification) - -## 12. 目标检测 - -目标检测任务的目标是给定一张图像或是视频帧，让计算机找出其中所有目标的位置，并给出每个目标的具体类别。对于人类来说，目标检测是一个非常简单的任务。然而，计算机能够“看到”的仅有一些值为0 ~ 255的矩阵，很难解图像或是视频帧中出现了人或是物体这样的高层语义概念，也就更加难以定位目标出现在图像中哪个区域。与此同时，由于目标会出现在图像或是视频帧中的任何位置，目标的形态千变万化，图像或是视频帧的背景千差万别，诸多因素都使得目标检测对计算机来说是一个具有挑战性的问题。 - -在目标检测任务中，我们介绍利用SSD方法完成目标检测。SSD全称：Single Shot MultiBox Detector，是目标检测领域较新且效果较好的检测算法之一，具有检测速度快且检测精度高的特点。 - -- 12.1 [Single Shot MultiBox Detector](https://github.com/PaddlePaddle/models/tree/develop/ssd/README.cn.md) - -## 13. 场景文字识别 - -许多场景图像中包含着丰富的文本信息，对理解图像信息有着重要作用，能够极大地帮助人们认知和理解场景图像的内容。场景文字识别是在图像背景复杂、分辨率低下、字体多样、分布随意等情况下，将图像信息转化为文字序列的过程，可认为是一种特别的翻译过程：将图像输入翻译为自然语言输出。场景图像文字识别技术的发展也促进了一些新型应用的产生，如通过自动识别路牌中的文字帮助街景应用获取更加准确的地址信息等。 - -在场景文字识别任务中，我们介绍如何将基于CNN的图像特征提取和基于RNN的序列翻译技术结合，免除人工定义特征，避免字符分割，使用自动学习到的图像特征，完成端到端地无约束字符定位和识别。 - -- 13.1 [场景文字识别](https://github.com/PaddlePaddle/models/tree/develop/scene_text_recognition) - -## 14. 语音识别 - -语音识别技术（Auto Speech Recognize，简称ASR）将人类语音中的词汇内容转化为计算机可读的输入，让机器能够“听懂”人类的语音，在语音助手、语音输入、语音交互等应用中发挥着重要作用。深度学习在语音识别领域取得了瞩目的成绩，端到端的深度学习方法将传统的声学模型、词典、语言模型等模块融为一个整体，不再依赖隐马尔可夫模型中的各种条件独立性假设，令模型变得更加简洁，一个神经网络模型以语音特征为输入，直接输出识别出的文本，目前已经成为语音识别最重要的手段。 - -在语音识别任务中，我们提供了基于 DeepSpeech2 模型的完整流水线，包括：特征提取、数据增强、模型训练、语言模型、解码模块等，并提供一个训练好的模型和体验实例，大家能够使用自己的声音来体验语音识别的乐趣。 - -14.1 [语音识别: DeepSpeech2](https://github.com/PaddlePaddle/DeepSpeech) - -本教程由[PaddlePaddle](https://github.com/PaddlePaddle/Paddle)创作，采用[Apache-2.0](LICENSE) 许可协议进行许可。 diff --git a/legacy/README.md b/legacy/README.md deleted file mode 100644 index f0719c1a26c04341e8de327143dc826248bb3607..0000000000000000000000000000000000000000 --- a/legacy/README.md +++ /dev/null @@ -1,89 +0,0 @@ - -# 该目录的模型已经不再维护，不推荐使用。建议使用Fluid目录下的模型。 - -# Introduction to models - -[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](https://github.com/PaddlePaddle/models) -[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](https://github.com/PaddlePaddle/models) -[![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE) - -PaddlePaddle provides a rich set of computational units to enable users to adopt a modular approach to solving various learning problems. In this repo, we demonstrate how to use PaddlePaddle to solve common machine learning tasks, providing several different neural network model that anyone can easily learn and use. - -## 1. Word Embedding - -The word embedding expresses words with a real vector. Each dimension of the vector represents some of the latent grammatical or semantic features of the text and is one of the most successful concepts in the field of natural language processing. The generalized word vector can also be applied to discrete features. The study of word vector is usually an unsupervised learning. Therefore, it is possible to take full advantage of massive unmarked data to capture the relationship between features and to solve the problem of sparse features, missing tag data, and data noise. However, in the common word vector learning method, the last layer of the model often encounters a large-scale classification problem, which is the bottleneck of computing performance. - -In the example of word vectors, we show how to use Hierarchical-Sigmoid and Noise Contrastive Estimation (NCE) to accelerate word-vector learning. - -- 1.1 [Hsigmoid Accelerated Word Vector Training](https://github.com/PaddlePaddle/models/tree/develop/legacy/hsigmoid) -- 1.2 [Noise Contrastive Estimation Accelerated Word Vector Training](https://github.com/PaddlePaddle/models/tree/develop/legacy/nce_cost) - - -## 2. RNN language model - -The language model is important in the field of natural language processing. In addition to getting the word vector (a by-product of language model training), it can also help us to generate text. Given a number of words, the language model can help us predict the next most likely word. In the example of using the language model to generate text, we focus on the recurrent neural network language model. We can use the instructions in the document quickly adapt to their training corpus, complete automatic writing poetry, automatic writing prose and other interesting models. - -- 2.1 [Generate text using the RNN language model](https://github.com/PaddlePaddle/models/tree/develop/legacy/generate_sequence_by_rnn_lm) - -## 3. Click-Through Rate prediction -The click-through rate model predicts the probability that a user will click on an ad. This is widely used for advertising technology. Logistic Regression has a good learning performance for large-scale sparse features in the early stages of the development of click-through rate prediction. In recent years, DNN model because of its strong learning ability to gradually take the banner rate of the task of the banner. - -In the example of click-through rate estimates, we first give the Google's Wide & Deep model. This model combines the advantages of DNN and the applicable logistic regression model for DNN and large-scale sparse features. Then we provide the deep factorization machine for click-through rate prediction. The deep factorization machine combines the factorization machine and deep neural networks to model both low order and high order interactions of input features. - -- 3.1 [Click-Through Rate Model](https://github.com/PaddlePaddle/models/tree/develop/legacy/ctr) -- 3.2 [Deep Factorization Machine for Click-Through Rate prediction](https://github.com/PaddlePaddle/models/tree/develop/legacy/deep_fm) - -## 4. Text classification - -Text classification is one of the most basic tasks in natural language processing. The deep learning method can eliminate the complex feature engineering, and use the original text as input to optimize the classification accuracy. - -For text classification, we provide a non-sequential text classification model based on DNN and CNN. (For LSTM-based model, please refer to PaddleBook [Sentiment Analysis](http://www.paddlepaddle.org/docs/develop/book/06.understand_sentiment/index.html)). - -- 4.1 [Sentiment analysis based on DNN / CNN](https://github.com/PaddlePaddle/models/tree/develop/legacy/text_classification) - -## 5. Learning to rank - -Learning to rank (LTR) is one of the core problems in information retrieval and search engine research. Training data is used by a learning algorithm to produce a ranking model which computes the relevance of documents for actual queries. -The depth neural network can be used to model the fractional function to form various LTR models based on depth learning. - -The algorithms for learning to rank are usually categorized into three groups by their input representation and the loss function. These are pointwise, pairwise and listwise approaches. Here we demonstrate RankLoss loss function method (pairwise approach), and LambdaRank loss function method (listwise approach). (For Pointwise approaches, please refer to [Recommended System](http://www.paddlepaddle.org/docs/develop/book/05.recommender_system/index.html)). - -- 5.1 [Learning to rank based on Pairwise and Listwise approches](https://github.com/PaddlePaddle/models/tree/develop/legacy/ltr) - -## 6. Semantic model -The deep structured semantic model uses the DNN model to learn the vector representation of the low latitude in a continuous semantic space, finally models the semantic similarity between the two sentences. - -In this example, we demonstrate how to use PaddlePaddle to implement a generic deep structured semantic model to model the semantic similarity between two strings. The model supports different network structures such as CNN (Convolutional Network), FC (Fully Connected Network), RNN (Recurrent Neural Network), and different loss functions such as classification, regression, and sequencing. - -- 6.1 [Deep structured semantic model](https://github.com/PaddlePaddle/models/tree/develop/legacy/dssm) - -## 7. Sequence tagging - -Given the input sequence, the sequence tagging model is one of the most basic tasks in the natural language processing by assigning a category tag to each element in the sequence. Recurrent neural network models with Conditional Random Field (CRF) are commonly used for sequence tagging tasks. - -In the example of the sequence tagging, we describe how to train an end-to-end sequence tagging model with the Named Entity Recognition (NER) task as an example. - -- 7.1 [Name Entity Recognition](https://github.com/PaddlePaddle/models/tree/develop/legacy/sequence_tagging_for_ner) - -## 8. Sequence to sequence learning - -Sequence-to-sequence model has a wide range of applications. This includes machine translation, dialogue system, and parse tree generation. - -As an example for sequence-to-sequence learning, we take the machine translation task. We demonstrate the sequence-to-sequence mapping model without attention mechanism, which is the basis for all sequence-to-sequence learning models. We will use scheduled sampling to improve the problem of error accumulation in the RNN model, and machine translation with external memory mechanism. - -- 8.1 [Basic Sequence-to-sequence model](https://github.com/PaddlePaddle/models/tree/develop/legacy/nmt_without_attention) - -## 9. Image classification - -For the example of image classification, we show you how to train AlexNet, VGG, GoogLeNet, ResNet, Inception-v4, Inception-Resnet-V2 and Xception models in PaddlePaddle. It also provides model conversion tools that convert Caffe or TensorFlow trained model files into PaddlePaddle model files. - -- 9.1 [convert Caffe model file to PaddlePaddle model file](https://github.com/PaddlePaddle/models/tree/develop/legacy/image_classification/caffe2paddle) -- 9.2 [convert TensorFlow model file to PaddlePaddle model file](https://github.com/PaddlePaddle/models/tree/develop/legacy/image_classification/tf2paddle) -- 9.3 [AlexNet](https://github.com/PaddlePaddle/models/tree/develop/legacy/image_classification) -- 9.4 [VGG](https://github.com/PaddlePaddle/models/tree/develop/legacy/image_classification) -- 9.5 [Residual Network](https://github.com/PaddlePaddle/models/tree/develop/legacy/image_classification) -- 9.6 [Inception-v4](https://github.com/PaddlePaddle/models/tree/develop/legacy/image_classification) -- 9.7 [Inception-Resnet-V2](https://github.com/PaddlePaddle/models/tree/develop/legacy/image_classification) -- 9.8 [Xception](https://github.com/PaddlePaddle/models/tree/develop/legacy/image_classification) - -This tutorial is contributed by [PaddlePaddle](https://github.com/PaddlePaddle/Paddle) and licensed under the [Apache-2.0 license](LICENSE). diff --git a/legacy/conv_seq2seq/README.md b/legacy/conv_seq2seq/README.md deleted file mode 100644 index 5b22c2c17ea2ff3588e93219e86d81a831242211..0000000000000000000000000000000000000000 --- a/legacy/conv_seq2seq/README.md +++ /dev/null @@ -1,70 +0,0 @@ -The minimum PaddlePaddle version needed for the code sample in this directory is v0.11.0. If you are on a version of PaddlePaddle earlier than v0.11.0, [please update your installation](http://www.paddlepaddle.org/docs/develop/documentation/en/build_and_install/pip_install_en.html). - ---- - -# Convolutional Sequence to Sequence Learning -This model implements the work in the following paper: - -Jonas Gehring, Micheal Auli, David Grangier, et al. Convolutional Sequence to Sequence Learning. Association for Computational Linguistics (ACL), 2017 - -# Data Preparation -- The data used in this tutorial can be downloaded by runing: - - ```bash - sh download.sh - ``` - -- Each line in the data file contains one sample and each sample consists of a source sentence and a target sentence. And the two sentences are seperated by '\t'. So, to use your own data, it should be organized as follows: - - ``` - \t - ``` - -# Training a Model -- Modify the following script if needed and then run: - - ```bash - python train.py \ - --train_data_path ./data/train \ - --test_data_path ./data/test \ - --src_dict_path ./data/src_dict \ - --trg_dict_path ./data/trg_dict \ - --enc_blocks "[(256, 3)] * 5" \ - --dec_blocks "[(256, 3)] * 3" \ - --emb_size 256 \ - --pos_size 200 \ - --drop_rate 0.2 \ - --use_bn False \ - --use_gpu False \ - --trainer_count 1 \ - --batch_size 32 \ - --num_passes 20 \ - >train.log 2>&1 - ``` - -# Inferring by a Trained Model -- Infer by a trained model by running: - - ```bash - python infer.py \ - --infer_data_path ./data/dev \ - --src_dict_path ./data/src_dict \ - --trg_dict_path ./data/trg_dict \ - --enc_blocks "[(256, 3)] * 5" \ - --dec_blocks "[(256, 3)] * 3" \ - --emb_size 256 \ - --pos_size 200 \ - --drop_rate 0.2 \ - --use_bn False \ - --use_gpu False \ - --trainer_count 1 \ - --max_len 100 \ - --batch_size 256 \ - --beam_size 1 \ - --is_show_attention False \ - --model_path ./params.pass-0.tar.gz \ - 1>infer_result 2>infer.log - ``` - -# Notes -Since PaddlePaddle of current version doesn't support weight normalization, we use batch normalization instead to confirm convergence when the network is deep. diff --git a/legacy/conv_seq2seq/beamsearch.py b/legacy/conv_seq2seq/beamsearch.py deleted file mode 100644 index dd8562f018c803d4f0d7bbba4a2a006ece904851..0000000000000000000000000000000000000000 --- a/legacy/conv_seq2seq/beamsearch.py +++ /dev/null @@ -1,197 +0,0 @@ -#coding=utf-8 - -import sys -import time -import math -import numpy as np - -import reader - - -class BeamSearch(object): - """ - Generate sequence by beam search - """ - - def __init__(self, - inferer, - trg_dict, - pos_size, - padding_num, - batch_size=1, - beam_size=1, - max_len=100): - self.inferer = inferer - self.trg_dict = trg_dict - self.reverse_trg_dict = reader.get_reverse_dict(trg_dict) - self.word_padding = trg_dict.__len__() - self.pos_size = pos_size - self.pos_padding = pos_size - self.padding_num = padding_num - self.win_len = padding_num + 1 - self.max_len = max_len - self.batch_size = batch_size - self.beam_size = beam_size - - def get_beam_input(self, batch, sample_list): - """ - Get input for generation at the current iteration. - """ - beam_input = [] - - for sample_id in sample_list: - for path in self.candidate_path[sample_id]: - if len(path['seq']) < self.win_len: - cur_trg = [self.word_padding] * ( - self.win_len - len(path['seq']) - 1 - ) + [self.trg_dict['']] + path['seq'] - cur_trg_pos = [self.pos_padding] * ( - self.win_len - len(path['seq']) - 1) + [0] + range( - 1, len(path['seq']) + 1) - else: - cur_trg = path['seq'][-self.win_len:] - cur_trg_pos = range( - len(path['seq']) + 1 - self.win_len, - len(path['seq']) + 1) - - beam_input.append(batch[sample_id] + [cur_trg] + [cur_trg_pos]) - - return beam_input - - def get_prob(self, beam_input): - """ - Get the probabilities of all possible tokens. - """ - row_list = [j * self.win_len for j in range(len(beam_input))] - prob = self.inferer.infer(beam_input, field='value')[row_list, :] - return prob - - def _top_k(self, prob, k): - """ - Get indices of the words with k highest probablities. - """ - return prob.argsort()[-k:][::-1] - - def beam_expand(self, prob, sample_list): - """ - In every iteration step, the model predicts the possible next words. - For each input sentence, the top beam_size words are selected as candidates. - """ - top_words = np.apply_along_axis(self._top_k, 1, prob, self.beam_size) - - candidate_words = [[]] * len(self.candidate_path) - idx = 0 - - for sample_id in sample_list: - for seq_id, path in enumerate(self.candidate_path[sample_id]): - for w in top_words[idx, :]: - score = path['score'] + math.log(prob[idx, w]) - candidate_words[sample_id] = candidate_words[sample_id] + [{ - 'word': w, - 'score': score, - 'seq_id': seq_id - }] - idx = idx + 1 - - return candidate_words - - def beam_shrink(self, candidate_words, sample_list): - """ - Pruning process of the beam search. During the process, beam_size most post possible - sequences are selected for the beam in the next generation. - """ - new_path = [[]] * len(self.candidate_path) - - for sample_id in sample_list: - beam_words = sorted( - candidate_words[sample_id], - key=lambda x: x['score'], - reverse=True)[:self.beam_size] - - complete_seq_min_score = None - complete_path_num = len(self.complete_path[sample_id]) - - if complete_path_num > 0: - complete_seq_min_score = min(self.complete_path[sample_id], - key=lambda x: x['score'])['score'] - if complete_path_num >= self.beam_size: - beam_words_max_score = beam_words[0]['score'] - if beam_words_max_score < complete_seq_min_score: - continue - - for w in beam_words: - - if w['word'] == self.trg_dict['']: - if complete_path_num < self.beam_size or complete_seq_min_score <= w[ - 'score']: - - seq = self.candidate_path[sample_id][w['seq_id']]['seq'] - self.complete_path[sample_id] = self.complete_path[ - sample_id] + [{ - 'seq': seq, - 'score': w['score'] - }] - - if complete_seq_min_score is None or complete_seq_min_score > w[ - 'score']: - complete_seq_min_score = w['score'] - else: - seq = self.candidate_path[sample_id][w['seq_id']]['seq'] + [ - w['word'] - ] - new_path[sample_id] = new_path[sample_id] + [{ - 'seq': seq, - 'score': w['score'] - }] - - return new_path - - def search_one_batch(self, batch): - """ - Perform beam search on one mini-batch. - """ - real_size = len(batch) - self.candidate_path = [[{'seq': [], 'score': 0.}]] * real_size - self.complete_path = [[]] * real_size - sample_list = range(real_size) - - for i in xrange(self.max_len): - beam_input = self.get_beam_input(batch, sample_list) - prob = self.get_prob(beam_input) - - candidate_words = self.beam_expand(prob, sample_list) - new_path = self.beam_shrink(candidate_words, sample_list) - self.candidate_path = new_path - sample_list = [ - sample_id for sample_id in sample_list - if len(new_path[sample_id]) > 0 - ] - - if len(sample_list) == 0: - break - - final_path = [] - for i in xrange(real_size): - top_path = sorted( - self.complete_path[i] + self.candidate_path[i], - key=lambda x: x['score'], - reverse=True)[:self.beam_size] - final_path.append(top_path) - return final_path - - def search(self, infer_data): - """ - Perform beam search on all data. - """ - - def _to_sentence(seq): - raw_sentence = [self.reverse_trg_dict[id] for id in seq] - sentence = " ".join(raw_sentence) - return sentence - - for pos in xrange(0, len(infer_data), self.batch_size): - batch = infer_data[pos:min(pos + self.batch_size, len(infer_data))] - self.final_path = self.search_one_batch(batch) - for top_path in self.final_path: - print _to_sentence(top_path[0]['seq']) - sys.stdout.flush() diff --git a/legacy/conv_seq2seq/download.sh b/legacy/conv_seq2seq/download.sh deleted file mode 100644 index b1a924d25b1a10ade9f4be8b504933d1efa01905..0000000000000000000000000000000000000000 --- a/legacy/conv_seq2seq/download.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -CUR_PATH=`pwd` -git clone https://github.com/moses-smt/mosesdecoder.git -git clone https://github.com/rizar/actor-critic-public - -export MOSES=`pwd`/mosesdecoder -export LVSR=`pwd`/actor-critic-public - -cd actor-critic-public/exp/ted -sh create_dataset.sh - -cd $CUR_PATH -mkdir data -cp actor-critic-public/exp/ted/prep/*-* data/ -cp actor-critic-public/exp/ted/vocab.* data/ - -cd data -python ../preprocess.py - -cd .. -rm -rf actor-critic-public mosesdecoder diff --git a/legacy/conv_seq2seq/infer.py b/legacy/conv_seq2seq/infer.py deleted file mode 100644 index c804a84e71ffe920b72064cb05461d72c444ac73..0000000000000000000000000000000000000000 --- a/legacy/conv_seq2seq/infer.py +++ /dev/null @@ -1,236 +0,0 @@ -#coding=utf-8 - -import sys -import argparse -import distutils.util -import gzip - -import paddle.v2 as paddle -from model import conv_seq2seq -from beamsearch import BeamSearch -import reader - - -def parse_args(): - parser = argparse.ArgumentParser( - description="PaddlePaddle Convolutional Seq2Seq") - parser.add_argument( - '--infer_data_path', - type=str, - required=True, - help="Path of the dataset for inference") - parser.add_argument( - '--src_dict_path', - type=str, - required=True, - help='Path of the source dictionary') - parser.add_argument( - '--trg_dict_path', - type=str, - required=True, - help='path of the target dictionary') - parser.add_argument( - '--enc_blocks', type=str, help='Convolution blocks of the encoder') - parser.add_argument( - '--dec_blocks', type=str, help='Convolution blocks of the decoder') - parser.add_argument( - '--emb_size', - type=int, - default=256, - help='Dimension of word embedding. (default: %(default)s)') - parser.add_argument( - '--pos_size', - type=int, - default=200, - help='Total number of the position indexes. (default: %(default)s)') - parser.add_argument( - '--drop_rate', - type=float, - default=0., - help='Dropout rate. (default: %(default)s)') - parser.add_argument( - "--use_bn", - default=False, - type=distutils.util.strtobool, - help="Use batch normalization or not. (default: %(default)s)") - parser.add_argument( - "--use_gpu", - default=False, - type=distutils.util.strtobool, - help="Use gpu or not. (default: %(default)s)") - parser.add_argument( - "--trainer_count", - default=1, - type=int, - help="Trainer number. (default: %(default)s)") - parser.add_argument( - '--max_len', - type=int, - default=100, - help="The maximum length of the sentence to be generated. (default: %(default)s)" - ) - parser.add_argument( - "--batch_size", - default=1, - type=int, - help="Size of a mini-batch. (default: %(default)s)") - parser.add_argument( - "--beam_size", - default=1, - type=int, - help="The width of beam expansion. (default: %(default)s)") - parser.add_argument( - "--model_path", - type=str, - required=True, - help="The path of trained model. (default: %(default)s)") - parser.add_argument( - "--is_show_attention", - default=False, - type=distutils.util.strtobool, - help="Whether to show attention weight or not. (default: %(default)s)") - return parser.parse_args() - - -def infer(infer_data_path, - src_dict_path, - trg_dict_path, - model_path, - enc_conv_blocks, - dec_conv_blocks, - emb_dim=256, - pos_size=200, - drop_rate=0., - use_bn=False, - max_len=100, - batch_size=1, - beam_size=1, - is_show_attention=False): - """ - Inference. - - :param infer_data_path: The path of the data for inference. - :type infer_data_path: str - :param src_dict_path: The path of the source dictionary. - :type src_dict_path: str - :param trg_dict_path: The path of the target dictionary. - :type trg_dict_path: str - :param model_path: The path of a trained model. - :type model_path: str - :param enc_conv_blocks: The scale list of the encoder's convolution blocks. And each element of - the list contains output dimension and context length of the corresponding - convolution block. - :type enc_conv_blocks: list of tuple - :param dec_conv_blocks: The scale list of the decoder's convolution blocks. And each element of - the list contains output dimension and context length of the corresponding - convolution block. - :type dec_conv_blocks: list of tuple - :param emb_dim: The dimension of the embedding vector. - :type emb_dim: int - :param pos_size: The total number of the position indexes, which means - the maximum value of the index is pos_size - 1. - :type pos_size: int - :param drop_rate: Dropout rate. - :type drop_rate: float - :param use_bn: Whether to use batch normalization or not. False is the default value. - :type use_bn: bool - :param max_len: The maximum length of the sentence to be generated. - :type max_len: int - :param beam_size: The width of beam expansion. - :type beam_size: int - :param is_show_attention: Whether to show attention weight or not. False is the default value. - :type is_show_attention: bool - """ - # load dict - src_dict = reader.load_dict(src_dict_path) - trg_dict = reader.load_dict(trg_dict_path) - src_dict_size = src_dict.__len__() - trg_dict_size = trg_dict.__len__() - - prob, weight = conv_seq2seq( - src_dict_size=src_dict_size, - trg_dict_size=trg_dict_size, - pos_size=pos_size, - emb_dim=emb_dim, - enc_conv_blocks=enc_conv_blocks, - dec_conv_blocks=dec_conv_blocks, - drop_rate=drop_rate, - with_bn=use_bn, - is_infer=True) - - # load parameters - parameters = paddle.parameters.Parameters.from_tar(gzip.open(model_path)) - - padding_list = [context_len - 1 for (size, context_len) in dec_conv_blocks] - padding_num = reduce(lambda x, y: x + y, padding_list) - infer_reader = reader.data_reader( - data_file=infer_data_path, - src_dict=src_dict, - trg_dict=trg_dict, - pos_size=pos_size, - padding_num=padding_num) - - if is_show_attention: - attention_inferer = paddle.inference.Inference( - output_layer=weight, parameters=parameters) - for i, data in enumerate(infer_reader()): - src_len = len(data[0]) - trg_len = len(data[2]) - attention_weight = attention_inferer.infer( - [data], field='value', flatten_result=False) - attention_weight = [ - weight.reshape((trg_len, src_len)) - for weight in attention_weight - ] - print attention_weight - break - return - - infer_data = [] - for i, raw_data in enumerate(infer_reader()): - infer_data.append([raw_data[0], raw_data[1]]) - - inferer = paddle.inference.Inference( - output_layer=prob, parameters=parameters) - - searcher = BeamSearch( - inferer=inferer, - trg_dict=trg_dict, - pos_size=pos_size, - padding_num=padding_num, - max_len=max_len, - batch_size=batch_size, - beam_size=beam_size) - - searcher.search(infer_data) - return - - -def main(): - args = parse_args() - enc_conv_blocks = eval(args.enc_blocks) - dec_conv_blocks = eval(args.dec_blocks) - - sys.setrecursionlimit(10000) - - paddle.init(use_gpu=args.use_gpu, trainer_count=args.trainer_count) - - infer( - infer_data_path=args.infer_data_path, - src_dict_path=args.src_dict_path, - trg_dict_path=args.trg_dict_path, - model_path=args.model_path, - enc_conv_blocks=enc_conv_blocks, - dec_conv_blocks=dec_conv_blocks, - emb_dim=args.emb_size, - pos_size=args.pos_size, - drop_rate=args.drop_rate, - use_bn=args.use_bn, - max_len=args.max_len, - batch_size=args.batch_size, - beam_size=args.beam_size, - is_show_attention=args.is_show_attention) - - -if __name__ == '__main__': - main() diff --git a/legacy/conv_seq2seq/model.py b/legacy/conv_seq2seq/model.py deleted file mode 100644 index c31238f83172fdc3d6240095279d1c953ab272ae..0000000000000000000000000000000000000000 --- a/legacy/conv_seq2seq/model.py +++ /dev/null @@ -1,440 +0,0 @@ -#coding=utf-8 - -import math - -import paddle.v2 as paddle - -__all__ = ["conv_seq2seq"] - - -def gated_conv_with_batchnorm(input, - size, - context_len, - context_start=None, - learning_rate=1.0, - drop_rate=0., - with_bn=False): - """ - Definition of the convolution block. - - :param input: The input of this block. - :type input: LayerOutput - :param size: The dimension of the block's output. - :type size: int - :param context_len: The context length of the convolution. - :type context_len: int - :param context_start: The start position of the context. - :type context_start: int - :param learning_rate: The learning rate factor of the parameters in the block. - The actual learning rate is the product of the global - learning rate and this factor. - :type learning_rate: float - :param drop_rate: Dropout rate. - :type drop_rate: float - :param with_bn: Whether to use batch normalization or not. False is the default - value. - :type with_bn: bool - :return: The output of the convolution block. - :rtype: LayerOutput - """ - input = paddle.layer.dropout(input=input, dropout_rate=drop_rate) - - context = paddle.layer.mixed( - size=input.size * context_len, - input=paddle.layer.context_projection( - input=input, context_len=context_len, context_start=context_start)) - - raw_conv = paddle.layer.fc( - input=context, - size=size * 2, - act=paddle.activation.Linear(), - param_attr=paddle.attr.Param( - initial_mean=0., - initial_std=math.sqrt(4.0 * (1.0 - drop_rate) / context.size), - learning_rate=learning_rate), - bias_attr=False) - - if with_bn: - raw_conv = paddle.layer.batch_norm( - input=raw_conv, - act=paddle.activation.Linear(), - param_attr=paddle.attr.Param(learning_rate=learning_rate)) - - with paddle.layer.mixed(size=size) as conv: - conv += paddle.layer.identity_projection(raw_conv, size=size, offset=0) - - with paddle.layer.mixed(size=size, act=paddle.activation.Sigmoid()) as gate: - gate += paddle.layer.identity_projection( - raw_conv, size=size, offset=size) - - with paddle.layer.mixed(size=size) as gated_conv: - gated_conv += paddle.layer.dotmul_operator(conv, gate) - - return gated_conv - - -def encoder(token_emb, - pos_emb, - conv_blocks=[(256, 3)] * 5, - num_attention=3, - drop_rate=0., - with_bn=False): - """ - Definition of the encoder. - - :param token_emb: The embedding vector of the input token. - :type token_emb: LayerOutput - :param pos_emb: The embedding vector of the input token's position. - :type pos_emb: LayerOutput - :param conv_blocks: The scale list of the convolution blocks. Each element of - the list contains output dimension and context length of - the corresponding convolution block. - :type conv_blocks: list of tuple - :param num_attention: The total number of the attention modules used in the decoder. - :type num_attention: int - :param drop_rate: Dropout rate. - :type drop_rate: float - :param with_bn: Whether to use batch normalization or not. False is the default - value. - :type with_bn: bool - :return: The input token encoding. - :rtype: LayerOutput - """ - embedding = paddle.layer.addto( - input=[token_emb, pos_emb], - layer_attr=paddle.attr.Extra(drop_rate=drop_rate)) - - proj_size = conv_blocks[0][0] - block_input = paddle.layer.fc( - input=embedding, - size=proj_size, - act=paddle.activation.Linear(), - param_attr=paddle.attr.Param( - initial_mean=0., - initial_std=math.sqrt((1.0 - drop_rate) / embedding.size), - learning_rate=1.0 / (2.0 * num_attention)), - bias_attr=True, ) - - for (size, context_len) in conv_blocks: - if block_input.size == size: - residual = block_input - else: - residual = paddle.layer.fc( - input=block_input, - size=size, - act=paddle.activation.Linear(), - param_attr=paddle.attr.Param(learning_rate=1.0 / - (2.0 * num_attention)), - bias_attr=True) - - gated_conv = gated_conv_with_batchnorm( - input=block_input, - size=size, - context_len=context_len, - learning_rate=1.0 / (2.0 * num_attention), - drop_rate=drop_rate, - with_bn=with_bn) - - with paddle.layer.mixed(size=size) as block_output: - block_output += paddle.layer.identity_projection(residual) - block_output += paddle.layer.identity_projection(gated_conv) - - # halve the variance of the sum - block_output = paddle.layer.slope_intercept( - input=block_output, slope=math.sqrt(0.5)) - - block_input = block_output - - emb_dim = embedding.size - encoded_vec = paddle.layer.fc( - input=block_output, - size=emb_dim, - act=paddle.activation.Linear(), - param_attr=paddle.attr.Param(learning_rate=1.0 / (2.0 * num_attention)), - bias_attr=True) - - encoded_sum = paddle.layer.addto(input=[encoded_vec, embedding]) - - # halve the variance of the sum - encoded_sum = paddle.layer.slope_intercept( - input=encoded_sum, slope=math.sqrt(0.5)) - - return encoded_vec, encoded_sum - - -def attention(decoder_state, cur_embedding, encoded_vec, encoded_sum): - """ - Definition of the attention. - - :param decoder_state: The hidden state of the decoder. - :type decoder_state: LayerOutput - :param cur_embedding: The embedding vector of the current token. - :type cur_embedding: LayerOutput - :param encoded_vec: The source token encoding. - :type encoded_vec: LayerOutput - :param encoded_sum: The sum of the source token's encoding and embedding. - :type encoded_sum: LayerOutput - :return: A context vector and the attention weight. - :rtype: LayerOutput - """ - residual = decoder_state - - state_size = decoder_state.size - emb_dim = cur_embedding.size - with paddle.layer.mixed(size=emb_dim, bias_attr=True) as state_summary: - state_summary += paddle.layer.full_matrix_projection(decoder_state) - state_summary += paddle.layer.identity_projection(cur_embedding) - - # halve the variance of the sum - state_summary = paddle.layer.slope_intercept( - input=state_summary, slope=math.sqrt(0.5)) - - expanded = paddle.layer.expand(input=state_summary, expand_as=encoded_vec) - - m = paddle.layer.dot_prod(input1=expanded, input2=encoded_vec) - - attention_weight = paddle.layer.fc(input=m, - size=1, - act=paddle.activation.SequenceSoftmax(), - bias_attr=False) - - scaled = paddle.layer.scaling(weight=attention_weight, input=encoded_sum) - - attended = paddle.layer.pooling( - input=scaled, pooling_type=paddle.pooling.Sum()) - - attended_proj = paddle.layer.fc(input=attended, - size=state_size, - act=paddle.activation.Linear(), - bias_attr=True) - - attention_result = paddle.layer.addto(input=[attended_proj, residual]) - - # halve the variance of the sum - attention_result = paddle.layer.slope_intercept( - input=attention_result, slope=math.sqrt(0.5)) - return attention_result, attention_weight - - -def decoder(token_emb, - pos_emb, - encoded_vec, - encoded_sum, - dict_size, - conv_blocks=[(256, 3)] * 3, - drop_rate=0., - with_bn=False): - """ - Definition of the decoder. - - :param token_emb: The embedding vector of the input token. - :type token_emb: LayerOutput - :param pos_emb: The embedding vector of the input token's position. - :type pos_emb: LayerOutput - :param encoded_vec: The source token encoding. - :type encoded_vec: LayerOutput - :param encoded_sum: The sum of the source token's encoding and embedding. - :type encoded_sum: LayerOutput - :param dict_size: The size of the target dictionary. - :type dict_size: int - :param conv_blocks: The scale list of the convolution blocks. Each element - of the list contains output dimension and context length - of the corresponding convolution block. - :type conv_blocks: list of tuple - :param drop_rate: Dropout rate. - :type drop_rate: float - :param with_bn: Whether to use batch normalization or not. False is the default - value. - :type with_bn: bool - :return: The probability of the predicted token and the attention weights. - :rtype: LayerOutput - """ - - def attention_step(decoder_state, cur_embedding, encoded_vec, encoded_sum): - conditional = attention( - decoder_state=decoder_state, - cur_embedding=cur_embedding, - encoded_vec=encoded_vec, - encoded_sum=encoded_sum) - return conditional - - embedding = paddle.layer.addto( - input=[token_emb, pos_emb], - layer_attr=paddle.attr.Extra(drop_rate=drop_rate)) - - proj_size = conv_blocks[0][0] - block_input = paddle.layer.fc( - input=embedding, - size=proj_size, - act=paddle.activation.Linear(), - param_attr=paddle.attr.Param( - initial_mean=0., - initial_std=math.sqrt((1.0 - drop_rate) / embedding.size)), - bias_attr=True, ) - - weight = [] - for (size, context_len) in conv_blocks: - if block_input.size == size: - residual = block_input - else: - residual = paddle.layer.fc(input=block_input, - size=size, - act=paddle.activation.Linear(), - bias_attr=True) - - decoder_state = gated_conv_with_batchnorm( - input=block_input, - size=size, - context_len=context_len, - context_start=0, - drop_rate=drop_rate, - with_bn=with_bn) - - group_inputs = [ - decoder_state, - embedding, - paddle.layer.StaticInput(input=encoded_vec), - paddle.layer.StaticInput(input=encoded_sum), - ] - - conditional, attention_weight = paddle.layer.recurrent_group( - step=attention_step, input=group_inputs) - weight.append(attention_weight) - - block_output = paddle.layer.addto(input=[conditional, residual]) - - # halve the variance of the sum - block_output = paddle.layer.slope_intercept( - input=block_output, slope=math.sqrt(0.5)) - - block_input = block_output - - out_emb_dim = embedding.size - block_output = paddle.layer.fc( - input=block_output, - size=out_emb_dim, - act=paddle.activation.Linear(), - layer_attr=paddle.attr.Extra(drop_rate=drop_rate)) - - decoder_out = paddle.layer.fc( - input=block_output, - size=dict_size, - act=paddle.activation.Softmax(), - param_attr=paddle.attr.Param( - initial_mean=0., - initial_std=math.sqrt((1.0 - drop_rate) / block_output.size)), - bias_attr=True) - - return decoder_out, weight - - -def conv_seq2seq(src_dict_size, - trg_dict_size, - pos_size, - emb_dim, - enc_conv_blocks=[(256, 3)] * 5, - dec_conv_blocks=[(256, 3)] * 3, - drop_rate=0., - with_bn=False, - is_infer=False): - """ - Definition of convolutional sequence-to-sequence network. - - :param src_dict_size: The size of the source dictionary. - :type src_dict_size: int - :param trg_dict_size: The size of the target dictionary. - :type trg_dict_size: int - :param pos_size: The total number of the position indexes, which means - the maximum value of the index is pos_size - 1. - :type pos_size: int - :param emb_dim: The dimension of the embedding vector. - :type emb_dim: int - :param enc_conv_blocks: The scale list of the encoder's convolution blocks. Each element - of the list contains output dimension and context length of the - corresponding convolution block. - :type enc_conv_blocks: list of tuple - :param dec_conv_blocks: The scale list of the decoder's convolution blocks. Each element - of the list contains output dimension and context length of the - corresponding convolution block. - :type dec_conv_blocks: list of tuple - :param drop_rate: Dropout rate. - :type drop_rate: float - :param with_bn: Whether to use batch normalization or not. False is the default value. - :type with_bn: bool - :param is_infer: Whether infer or not. - :type is_infer: bool - :return: Cost or output layer. - :rtype: LayerOutput - """ - src = paddle.layer.data( - name='src_word', - type=paddle.data_type.integer_value_sequence(src_dict_size)) - src_pos = paddle.layer.data( - name='src_word_pos', - type=paddle.data_type.integer_value_sequence(pos_size + - 1)) # one for padding - - src_emb = paddle.layer.embedding( - input=src, - size=emb_dim, - name='src_word_emb', - param_attr=paddle.attr.Param( - initial_mean=0., initial_std=0.1)) - src_pos_emb = paddle.layer.embedding( - input=src_pos, - size=emb_dim, - name='src_pos_emb', - param_attr=paddle.attr.Param( - initial_mean=0., initial_std=0.1)) - - num_attention = len(dec_conv_blocks) - encoded_vec, encoded_sum = encoder( - token_emb=src_emb, - pos_emb=src_pos_emb, - conv_blocks=enc_conv_blocks, - num_attention=num_attention, - drop_rate=drop_rate, - with_bn=with_bn) - - trg = paddle.layer.data( - name='trg_word', - type=paddle.data_type.integer_value_sequence(trg_dict_size + - 1)) # one for padding - trg_pos = paddle.layer.data( - name='trg_word_pos', - type=paddle.data_type.integer_value_sequence(pos_size + - 1)) # one for padding - - trg_emb = paddle.layer.embedding( - input=trg, - size=emb_dim, - name='trg_word_emb', - param_attr=paddle.attr.Param( - initial_mean=0., initial_std=0.1)) - trg_pos_emb = paddle.layer.embedding( - input=trg_pos, - size=emb_dim, - name='trg_pos_emb', - param_attr=paddle.attr.Param( - initial_mean=0., initial_std=0.1)) - - decoder_out, weight = decoder( - token_emb=trg_emb, - pos_emb=trg_pos_emb, - encoded_vec=encoded_vec, - encoded_sum=encoded_sum, - dict_size=trg_dict_size, - conv_blocks=dec_conv_blocks, - drop_rate=drop_rate, - with_bn=with_bn) - - if is_infer: - return decoder_out, weight - - trg_next_word = paddle.layer.data( - name='trg_next_word', - type=paddle.data_type.integer_value_sequence(trg_dict_size)) - cost = paddle.layer.classification_cost( - input=decoder_out, label=trg_next_word) - - return cost diff --git a/legacy/conv_seq2seq/preprocess.py b/legacy/conv_seq2seq/preprocess.py deleted file mode 100644 index 1d5c7cdd7b5cc91e28854fa0bbeeffc9dcbe4e5c..0000000000000000000000000000000000000000 --- a/legacy/conv_seq2seq/preprocess.py +++ /dev/null @@ -1,30 +0,0 @@ -#coding=utf-8 - -import cPickle - - -def concat_file(file1, file2, dst_file): - with open(dst_file, 'w') as dst: - with open(file1) as f1: - with open(file2) as f2: - for i, (line1, line2) in enumerate(zip(f1, f2)): - line1 = line1.strip() - line = line1 + '\t' + line2 - dst.write(line) - - -if __name__ == '__main__': - concat_file('dev.de-en.de', 'dev.de-en.en', 'dev') - concat_file('test.de-en.de', 'test.de-en.en', 'test') - concat_file('train.de-en.de', 'train.de-en.en', 'train') - - src_dict = cPickle.load(open('vocab.de')) - trg_dict = cPickle.load(open('vocab.en')) - - with open('src_dict', 'w') as f: - f.write('\n\nUNK\n') - f.writelines('\n'.join(src_dict.keys())) - - with open('trg_dict', 'w') as f: - f.write('\n\nUNK\n') - f.writelines('\n'.join(trg_dict.keys())) diff --git a/legacy/conv_seq2seq/reader.py b/legacy/conv_seq2seq/reader.py deleted file mode 100644 index ad420af5faade1cd5ee7ef947f7f8920ce6a8bdb..0000000000000000000000000000000000000000 --- a/legacy/conv_seq2seq/reader.py +++ /dev/null @@ -1,67 +0,0 @@ -#coding=utf-8 - -import random - - -def load_dict(dict_file): - word_dict = dict() - with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i - return word_dict - - -def get_reverse_dict(dictionary): - reverse_dict = {dictionary[k]: k for k in dictionary.keys()} - return reverse_dict - - -def load_data(data_file, src_dict, trg_dict): - UNK_IDX = src_dict['UNK'] - with open(data_file, 'r') as f: - for line in f: - line_split = line.strip().split('\t') - if len(line_split) < 2: - continue - src, trg = line_split - src_words = src.strip().split() - trg_words = trg.strip().split() - src_seq = [src_dict.get(w, UNK_IDX) for w in src_words] - trg_seq = [trg_dict.get(w, UNK_IDX) for w in trg_words] - yield src_seq, trg_seq - - -def data_reader(data_file, src_dict, trg_dict, pos_size, padding_num): - def reader(): - UNK_IDX = src_dict['UNK'] - word_padding = trg_dict.__len__() - pos_padding = pos_size - - def _get_pos(pos_list, pos_size, pos_padding): - return [pos if pos < pos_size else pos_padding for pos in pos_list] - - with open(data_file, 'r') as f: - for line in f: - line_split = line.strip().split('\t') - if len(line_split) != 2: - continue - src, trg = line_split - src = src.strip().split() - src_word = [src_dict.get(w, UNK_IDX) for w in src] - src_word_pos = range(len(src_word)) - src_word_pos = _get_pos(src_word_pos, pos_size, pos_padding) - - trg = trg.strip().split() - trg_word = [trg_dict[''] - ] + [trg_dict.get(w, UNK_IDX) for w in trg] - trg_word_pos = range(len(trg_word)) - trg_word_pos = _get_pos(trg_word_pos, pos_size, pos_padding) - - trg_next_word = trg_word[1:] + [trg_dict['']] - trg_word = [word_padding] * padding_num + trg_word - trg_word_pos = [pos_padding] * padding_num + trg_word_pos - trg_next_word = trg_next_word + [trg_dict['']] * padding_num - yield src_word, src_word_pos, trg_word, trg_word_pos, trg_next_word - - return reader diff --git a/legacy/conv_seq2seq/train.py b/legacy/conv_seq2seq/train.py deleted file mode 100644 index 4bd9a1af675ada5820bb375938a4675e6e71fbe1..0000000000000000000000000000000000000000 --- a/legacy/conv_seq2seq/train.py +++ /dev/null @@ -1,263 +0,0 @@ -#coding=utf-8 - -import os -import sys -import time -import argparse -import distutils.util -import gzip -import numpy as np - -import paddle.v2 as paddle -from model import conv_seq2seq -import reader - - -def parse_args(): - parser = argparse.ArgumentParser( - description="PaddlePaddle Convolutional Seq2Seq") - parser.add_argument( - '--train_data_path', - type=str, - required=True, - help="Path of the training set") - parser.add_argument( - '--test_data_path', type=str, help='Path of the test set') - parser.add_argument( - '--src_dict_path', - type=str, - required=True, - help='Path of source dictionary') - parser.add_argument( - '--trg_dict_path', - type=str, - required=True, - help='Path of target dictionary') - parser.add_argument( - '--enc_blocks', type=str, help='Convolution blocks of the encoder') - parser.add_argument( - '--dec_blocks', type=str, help='Convolution blocks of the decoder') - parser.add_argument( - '--emb_size', - type=int, - default=256, - help='Dimension of word embedding. (default: %(default)s)') - parser.add_argument( - '--pos_size', - type=int, - default=200, - help='Total number of the position indexes. (default: %(default)s)') - parser.add_argument( - '--drop_rate', - type=float, - default=0., - help='Dropout rate. (default: %(default)s)') - parser.add_argument( - "--use_bn", - default=False, - type=distutils.util.strtobool, - help="Use batch normalization or not. (default: %(default)s)") - parser.add_argument( - "--use_gpu", - default=False, - type=distutils.util.strtobool, - help="Use gpu or not. (default: %(default)s)") - parser.add_argument( - "--trainer_count", - default=1, - type=int, - help="Trainer number. (default: %(default)s)") - parser.add_argument( - '--batch_size', - type=int, - default=32, - help="Size of a mini-batch. (default: %(default)s)") - parser.add_argument( - '--num_passes', - type=int, - default=15, - help="Number of passes to train. (default: %(default)s)") - return parser.parse_args() - - -def create_reader(padding_num, - train_data_path, - test_data_path=None, - src_dict=None, - trg_dict=None, - pos_size=200, - batch_size=32): - - train_reader = paddle.batch( - reader=paddle.reader.shuffle( - reader=reader.data_reader( - data_file=train_data_path, - src_dict=src_dict, - trg_dict=trg_dict, - pos_size=pos_size, - padding_num=padding_num), - buf_size=10240), - batch_size=batch_size) - - test_reader = None - if test_data_path: - test_reader = paddle.batch( - reader=paddle.reader.shuffle( - reader=reader.data_reader( - data_file=test_data_path, - src_dict=src_dict, - trg_dict=trg_dict, - pos_size=pos_size, - padding_num=padding_num), - buf_size=10240), - batch_size=batch_size) - - return train_reader, test_reader - - -def train(train_data_path, - test_data_path, - src_dict_path, - trg_dict_path, - enc_conv_blocks, - dec_conv_blocks, - emb_dim=256, - pos_size=200, - drop_rate=0., - use_bn=False, - batch_size=32, - num_passes=15): - """ - Train the convolution sequence-to-sequence model. - - :param train_data_path: The path of the training set. - :type train_data_path: str - :param test_data_path: The path of the test set. - :type test_data_path: str - :param src_dict_path: The path of the source dictionary. - :type src_dict_path: str - :param trg_dict_path: The path of the target dictionary. - :type trg_dict_path: str - :param enc_conv_blocks: The scale list of the encoder's convolution blocks. And each element of - the list contains output dimension and context length of the corresponding - convolution block. - :type enc_conv_blocks: list of tuple - :param dec_conv_blocks: The scale list of the decoder's convolution blocks. And each element of - the list contains output dimension and context length of the corresponding - convolution block. - :type dec_conv_blocks: list of tuple - :param emb_dim: The dimension of the embedding vector. - :type emb_dim: int - :param pos_size: The total number of the position indexes, which means - the maximum value of the index is pos_size - 1. - :type pos_size: int - :param drop_rate: Dropout rate. - :type drop_rate: float - :param use_bn: Whether to use batch normalization or not. False is the default value. - :type use_bn: bool - :param batch_size: The size of a mini-batch. - :type batch_size: int - :param num_passes: The total number of the passes to train. - :type num_passes: int - """ - # load dict - src_dict = reader.load_dict(src_dict_path) - trg_dict = reader.load_dict(trg_dict_path) - src_dict_size = src_dict.__len__() - trg_dict_size = trg_dict.__len__() - - optimizer = paddle.optimizer.Adam(learning_rate=1e-3, ) - - cost = conv_seq2seq( - src_dict_size=src_dict_size, - trg_dict_size=trg_dict_size, - pos_size=pos_size, - emb_dim=emb_dim, - enc_conv_blocks=enc_conv_blocks, - dec_conv_blocks=dec_conv_blocks, - drop_rate=drop_rate, - with_bn=use_bn, - is_infer=False) - - # create parameters and trainer - parameters = paddle.parameters.create(cost) - trainer = paddle.trainer.SGD(cost=cost, - parameters=parameters, - update_equation=optimizer) - - padding_list = [context_len - 1 for (size, context_len) in dec_conv_blocks] - padding_num = reduce(lambda x, y: x + y, padding_list) - train_reader, test_reader = create_reader( - padding_num=padding_num, - train_data_path=train_data_path, - test_data_path=test_data_path, - src_dict=src_dict, - trg_dict=trg_dict, - pos_size=pos_size, - batch_size=batch_size) - - feeding = { - 'src_word': 0, - 'src_word_pos': 1, - 'trg_word': 2, - 'trg_word_pos': 3, - 'trg_next_word': 4 - } - - # create event handler - def event_handler(event): - if isinstance(event, paddle.event.EndIteration): - if event.batch_id % 20 == 0: - cur_time = time.strftime('%Y.%m.%d %H:%M:%S', time.localtime()) - print "[%s]: Pass: %d, Batch: %d, TrainCost: %f, %s" % ( - cur_time, event.pass_id, event.batch_id, event.cost, - event.metrics) - sys.stdout.flush() - - if isinstance(event, paddle.event.EndPass): - if test_reader is not None: - cur_time = time.strftime('%Y.%m.%d %H:%M:%S', time.localtime()) - result = trainer.test(reader=test_reader, feeding=feeding) - print "[%s]: Pass: %d, TestCost: %f, %s" % ( - cur_time, event.pass_id, result.cost, result.metrics) - sys.stdout.flush() - with gzip.open("output/params.pass-%d.tar.gz" % event.pass_id, - 'w') as f: - trainer.save_parameter_to_tar(f) - - if not os.path.exists('output'): - os.mkdir('output') - - trainer.train( - reader=train_reader, - event_handler=event_handler, - num_passes=num_passes, - feeding=feeding) - - -def main(): - args = parse_args() - enc_conv_blocks = eval(args.enc_blocks) - dec_conv_blocks = eval(args.dec_blocks) - - sys.setrecursionlimit(10000) - - paddle.init(use_gpu=args.use_gpu, trainer_count=args.trainer_count) - - train( - train_data_path=args.train_data_path, - test_data_path=args.test_data_path, - src_dict_path=args.src_dict_path, - trg_dict_path=args.trg_dict_path, - enc_conv_blocks=enc_conv_blocks, - dec_conv_blocks=dec_conv_blocks, - emb_dim=args.emb_size, - pos_size=args.pos_size, - drop_rate=args.drop_rate, - use_bn=args.use_bn, - batch_size=args.batch_size, - num_passes=args.num_passes) - - -if __name__ == '__main__': - main() diff --git a/legacy/ctr/README.cn.md b/legacy/ctr/README.cn.md deleted file mode 100644 index d717264c46529c4ca3be6500983558b0384a7d77..0000000000000000000000000000000000000000 --- a/legacy/ctr/README.cn.md +++ /dev/null @@ -1,369 +0,0 @@ -运行本目录下的程序示例需要使用PaddlePaddle v0.10.0 版本。如果您的PaddlePaddle安装版本低于此要求，请按照[安装文档](http://www.paddlepaddle.org/docs/develop/documentation/zh/build_and_install/pip_install_cn.html)中的说明更新PaddlePaddle安装版本。 - ---- - -# 点击率预估 - -以下是本例目录包含的文件以及对应说明: - -``` -├── README.md # 本教程markdown 文档 -├── dataset.md # 数据集处理教程 -├── images # 本教程图片目录 -│   ├── lr_vs_dnn.jpg -│   └── wide_deep.png -├── infer.py # 预测脚本 -├── network_conf.py # 模型网络配置 -├── reader.py # data reader -├── train.py # 训练脚本 -└── utils.py # helper functions -└── avazu_data_processer.py # 示例数据预处理脚本 -``` - -## 背景介绍 - -CTR(Click-Through Rate，点击率预估)\[[1](https://en.wikipedia.org/wiki/Click-through_rate)\] -是对用户点击一个特定链接的概率做出预测，是广告投放过程中的一个重要环节。精准的点击率预估对在线广告系统收益最大化具有重要意义。 - -当有多个广告位时，CTR 预估一般会作为排序的基准，比如在搜索引擎的广告系统里，当用户输入一个带商业价值的搜索词（query）时，系统大体上会执行下列步骤来展示广告： - -1. 获取与用户搜索词相关的广告集合 -2. 业务规则和相关性过滤 -3. 根据拍卖机制和 CTR 排序 -4. 展出广告 - -可以看到，CTR 在最终排序中起到了很重要的作用。 - -### 发展阶段 -在业内，CTR 模型经历了如下的发展阶段： - -- Logistic Regression(LR) / GBDT + 特征工程 -- LR + DNN 特征 -- DNN + 特征工程 - -在发展早期时 LR 一统天下，但最近 DNN 模型由于其强大的学习能力和逐渐成熟的性能优化， -逐渐地接过 CTR 预估任务的大旗。 - - -### LR vs DNN - -下图展示了 LR 和一个 \(3x2\) 的 DNN 模型的结构： - -

-
-Figure 1. LR 和 DNN 模型结构对比 -

- -LR 的蓝色箭头部分可以直接类比到 DNN 中对应的结构，可以看到 LR 和 DNN 有一些共通之处（比如权重累加）， -但前者的模型复杂度在相同输入维度下比后者可能低很多（从某方面讲，模型越复杂，越有潜力学习到更复杂的信息）； -如果 LR 要达到匹敌 DNN 的学习能力，必须增加输入的维度，也就是增加特征的数量， -这也就是为何 LR 和大规模的特征工程必须绑定在一起的原因。 - -LR 对于 DNN 模型的优势是对大规模稀疏特征的容纳能力，包括内存和计算量等方面，工业界都有非常成熟的优化方法； -而 DNN 模型具有自己学习新特征的能力，一定程度上能够提升特征使用的效率， -这使得 DNN 模型在同样规模特征的情况下，更有可能达到更好的学习效果。 - -本文后面的章节会演示如何使用 PaddlePaddle 编写一个结合两者优点的模型。 - - -## 数据和任务抽象 - -我们可以将 `click` 作为学习目标，任务可以有以下几种方案： - -1. 直接学习 click，0,1 作二元分类 -2. Learning to rank, 具体用 pairwise rank（标签 1>0）或者 listwise rank -3. 统计每个广告的点击率，将同一个 query 下的广告两两组合，点击率高的>点击率低的，做 rank 或者分类 - -我们直接使用第一种方法做分类任务。 - -我们使用 Kaggle 上 `Click-through rate prediction` 任务的数据集\[[2](https://www.kaggle.com/c/avazu-ctr-prediction/data)\] 来演示本例中的模型。 - -具体的特征处理方法参看 [data process](./dataset.md)。 - -本教程中演示模型的输入格式如下： - -``` -# \t \t click -1 23 190 \t 230:0.12 3421:0.9 23451:0.12 \t 0 -23 231 \t 1230:0.12 13421:0.9 \t 1 -``` - -详细的格式描述如下： - -- `dnn input ids` 采用 one-hot 表示，只需要填写值为1的ID（注意这里不是变长输入） -- `lr input sparse values` 使用了 `ID:VALUE` 的表示，值部分最好规约到值域 `[-1, 1]`。 - -此外，模型训练时需要传入一个文件描述 dnn 和 lr两个子模型的输入维度，文件的格式如下： - -``` -dnn_input_dim: -lr_input_dim: -``` - -其中， `` 表示一个整型数值。 - -本目录下的 `avazu_data_processor.py` 可以对下载的演示数据集\[[2](#参考文档)\] 进行处理，具体使用方法参考如下说明： - -``` -usage: avazu_data_processer.py [-h] --data_path DATA_PATH --output_dir - OUTPUT_DIR - [--num_lines_to_detect NUM_LINES_TO_DETECT] - [--test_set_size TEST_SET_SIZE] - [--train_size TRAIN_SIZE] - -PaddlePaddle CTR example - -optional arguments: - -h, --help show this help message and exit - --data_path DATA_PATH - path of the Avazu dataset - --output_dir OUTPUT_DIR - directory to output - --num_lines_to_detect NUM_LINES_TO_DETECT - number of records to detect dataset's meta info - --test_set_size TEST_SET_SIZE - size of the validation dataset(default: 10000) - --train_size TRAIN_SIZE - size of the trainset (default: 100000) -``` - -- `data_path` 是待处理的数据路径 -- `output_dir` 生成数据的输出路径 -- `num_lines_to_detect` 预先扫描数据生成ID的个数，这里是扫描的文件行数 -- `test_set_size` 生成测试集的行数 -- `train_size` 生成训练姐的行数 - -## Wide & Deep Learning Model - -谷歌在 16 年提出了 Wide & Deep Learning 的模型框架，用于融合适合学习抽象特征的 DNN 和 适用于大规模稀疏特征的 LR 两种模型的优点。 - - -### 模型简介 - -Wide & Deep Learning Model\[[3](#参考文献)\] 可以作为一种相对成熟的模型框架使用， -在 CTR 预估的任务中工业界也有一定的应用，因此本文将演示使用此模型来完成 CTR 预估的任务。 - -模型结构如下： - -

-
-Figure 2. Wide & Deep Model -

- -模型上边的 Wide 部分，可以容纳大规模系数特征，并且对一些特定的信息（比如 ID）有一定的记忆能力； -而模型下边的 Deep 部分，能够学习特征间的隐含关系，在相同数量的特征下有更好的学习和推导能力。 - - -### 编写模型输入 - -模型只接受 3 个输入，分别是 - -- `dnn_input` ，也就是 Deep 部分的输入 -- `lr_input` ，也就是 Wide 部分的输入 -- `click` ， 点击与否，作为二分类模型学习的标签 - -```python -dnn_merged_input = layer.data( - name='dnn_input', - type=paddle.data_type.sparse_binary_vector(data_meta_info['dnn_input'])) - -lr_merged_input = layer.data( - name='lr_input', - type=paddle.data_type.sparse_binary_vector(data_meta_info['lr_input'])) - -click = paddle.layer.data(name='click', type=dtype.dense_vector(1)) -``` - -### 编写 Wide 部分 - -Wide 部分直接使用了 LR 模型，但激活函数改成了 `RELU` 来加速 - -```python -def build_lr_submodel(): - fc = layer.fc( - input=lr_merged_input, size=1, name='lr', act=paddle.activation.Relu()) - return fc -``` - -### 编写 Deep 部分 - -Deep 部分使用了标准的多层前向传导的 DNN 模型 - -```python -def build_dnn_submodel(dnn_layer_dims): - dnn_embedding = layer.fc(input=dnn_merged_input, size=dnn_layer_dims[0]) - _input_layer = dnn_embedding - for i, dim in enumerate(dnn_layer_dims[1:]): - fc = layer.fc( - input=_input_layer, - size=dim, - act=paddle.activation.Relu(), - name='dnn-fc-%d' % i) - _input_layer = fc - return _input_layer -``` - -### 两者融合 - -两个 submodel 的最上层输出加权求和得到整个模型的输出，输出部分使用 `sigmoid` 作为激活函数，得到区间 (0,1) 的预测值， -来逼近训练数据中二元类别的分布，并最终作为 CTR 预估的值使用。 - -```python -# conbine DNN and LR submodels -def combine_submodels(dnn, lr): - merge_layer = layer.concat(input=[dnn, lr]) - fc = layer.fc( - input=merge_layer, - size=1, - name='output', - # use sigmoid function to approximate ctr, wihch is a float value between 0 and 1. - act=paddle.activation.Sigmoid()) - return fc -``` - -### 训练任务的定义 -```python -dnn = build_dnn_submodel(dnn_layer_dims) -lr = build_lr_submodel() -output = combine_submodels(dnn, lr) - -# ============================================================================== -# cost and train period -# ============================================================================== -classification_cost = paddle.layer.multi_binary_label_cross_entropy_cost( - input=output, label=click) - - -paddle.init(use_gpu=False, trainer_count=11) - -params = paddle.parameters.create(classification_cost) - -optimizer = paddle.optimizer.Momentum(momentum=0) - -trainer = paddle.trainer.SGD( - cost=classification_cost, parameters=params, update_equation=optimizer) - -dataset = AvazuDataset(train_data_path, n_records_as_test=test_set_size) - -def event_handler(event): - if isinstance(event, paddle.event.EndIteration): - if event.batch_id % 100 == 0: - logging.warning("Pass %d, Samples %d, Cost %f" % ( - event.pass_id, event.batch_id * batch_size, event.cost)) - - if event.batch_id % 1000 == 0: - result = trainer.test( - reader=paddle.batch(dataset.test, batch_size=1000), - feeding=field_index) - logging.warning("Test %d-%d, Cost %f" % (event.pass_id, event.batch_id, - result.cost)) - - -trainer.train( - reader=paddle.batch( - paddle.reader.shuffle(dataset.train, buf_size=500), - batch_size=batch_size), - feeding=field_index, - event_handler=event_handler, - num_passes=100) -``` -## 运行训练和测试 -训练模型需要如下步骤： - -1. 准备训练数据 - 1. 从 [Kaggle CTR](https://www.kaggle.com/c/avazu-ctr-prediction/data) 下载 train.gz - 2. 解压 train.gz 得到 train.txt - 3. `mkdir -p output; python avazu_data_processer.py --data_path train.txt --output_dir output --num_lines_to_detect 1000 --test_set_size 100` 生成演示数据 -2. 执行 `python train.py --train_data_path ./output/train.txt --test_data_path ./output/test.txt --data_meta_file ./output/data.meta.txt --model_type=0` 开始训练 - -上面第2个步骤可以为 `train.py` 填充命令行参数来定制模型的训练过程，具体的命令行参数及用法如下 - -``` -usage: train.py [-h] --train_data_path TRAIN_DATA_PATH - [--test_data_path TEST_DATA_PATH] [--batch_size BATCH_SIZE] - [--num_passes NUM_PASSES] - [--model_output_prefix MODEL_OUTPUT_PREFIX] --data_meta_file - DATA_META_FILE --model_type MODEL_TYPE - -PaddlePaddle CTR example - -optional arguments: - -h, --help show this help message and exit - --train_data_path TRAIN_DATA_PATH - path of training dataset - --test_data_path TEST_DATA_PATH - path of testing dataset - --batch_size BATCH_SIZE - size of mini-batch (default:10000) - --num_passes NUM_PASSES - number of passes to train - --model_output_prefix MODEL_OUTPUT_PREFIX - prefix of path for model to store (default: - ./ctr_models) - --data_meta_file DATA_META_FILE - path of data meta info file - --model_type MODEL_TYPE - model type, classification: 0, regression 1 (default - classification) -``` - -- `train_data_path` ： 训练集的路径 -- `test_data_path` : 测试集的路径 -- `num_passes`: 模型训练多少轮 -- `data_meta_file`: 参考[数据和任务抽象](### 数据和任务抽象)的描述。 -- `model_type`: 模型分类或回归 - - -## 用训好的模型做预测 -训好的模型可以用来预测新的数据， 预测数据的格式为 - -``` -# \t -1 23 190 \t 230:0.12 3421:0.9 23451:0.12 -23 231 \t 1230:0.12 13421:0.9 -``` - -这里与训练数据的格式唯一不同的地方，就是没有标签，也就是训练数据中第3列 `click` 对应的数值。 - -`infer.py` 的使用方法如下 - -``` -usage: infer.py [-h] --model_gz_path MODEL_GZ_PATH --data_path DATA_PATH - --prediction_output_path PREDICTION_OUTPUT_PATH - [--data_meta_path DATA_META_PATH] --model_type MODEL_TYPE - -PaddlePaddle CTR example - -optional arguments: - -h, --help show this help message and exit - --model_gz_path MODEL_GZ_PATH - path of model parameters gz file - --data_path DATA_PATH - path of the dataset to infer - --prediction_output_path PREDICTION_OUTPUT_PATH - path to output the prediction - --data_meta_path DATA_META_PATH - path of trainset's meta info, default is ./data.meta - --model_type MODEL_TYPE - model type, classification: 0, regression 1 (default - classification) -``` - -- `model_gz_path_model`：用 `gz` 压缩过的模型路径 -- `data_path` ： 需要预测的数据路径 -- `prediction_output_paht`：预测输出的路径 -- `data_meta_file` ：参考[数据和任务抽象](### 数据和任务抽象)的描述。 -- `model_type` ：分类或回归 - -示例数据可以用如下命令预测 - -``` -python infer.py --model_gz_path --data_path output/infer.txt --prediction_output_path predictions.txt --data_meta_path data.meta.txt -``` - -最终的预测结果位于 `predictions.txt`。 - -## 参考文献 -1. -2. -3. Cheng H T, Koc L, Harmsen J, et al. [Wide & deep learning for recommender systems](https://arxiv.org/pdf/1606.07792.pdf)[C]//Proceedings of the 1st Workshop on Deep Learning for Recommender Systems. ACM, 2016: 7-10. diff --git a/legacy/ctr/README.md b/legacy/ctr/README.md deleted file mode 100644 index 9ace483be6126b31e064ce3014cea1b08664f8cf..0000000000000000000000000000000000000000 --- a/legacy/ctr/README.md +++ /dev/null @@ -1,343 +0,0 @@ -The minimum PaddlePaddle version needed for the code sample in this directory is v0.10.0. If you are on a version of PaddlePaddle earlier than v0.10.0, [please update your installation](http://www.paddlepaddle.org/docs/develop/documentation/en/build_and_install/pip_install_en.html). - ---- - -# Click-Through Rate Prediction - -## Introduction - -CTR(Click-Through Rate)\[[1](https://en.wikipedia.org/wiki/Click-through_rate)\] -is a prediction of the probability that a user clicks on an advertisement. This model is widely used in the advertisement industry. Accurate click rate estimates are important for maximizing online advertising revenue. - -When there are multiple ad slots, CTR estimates are generally used as a baseline for ranking. For example, in a search engine's ad system, when the user enters a query, the system typically performs the following steps to show relevant ads. - -1. Get the ad collection associated with the user's search term. -2. Business rules and relevance filtering. -3. Rank by auction mechanism and CTR. -4. Show ads. - -Here，CTR plays a crucial role. - -### Brief history -Historically, the CTR prediction model has been evolving as follows. - -- Logistic Regression(LR) / Gradient Boosting Decision Trees (GBDT) + feature engineering -- LR + Deep Neural Network (DNN) -- DNN + feature engineering - -In the early stages of development LR dominated, but the recent years DNN based models are mainly used. - - -### LR vs DNN - -The following figure shows the structure of LR and DNN model: - -

-
-Figure 1. LR and DNN model structure comparison -

- -We can see, LR and CNN have some common structures. However, DNN can have non-linear relation between input and output values by adding activation unit and further layers. This enables DNN to achieve better learning results in CTR estimates. - -In the following, we demonstrate how to use PaddlePaddle to learn to predict CTR. - -## Data and Model formation - -Here `click` is the learning objective. There are several ways to learn the objectives. - -1. Direct learning click, 0,1 for binary classification -2. Learning to rank, pairwise rank or listwise rank -3. Measure the ad click rate of each ad, then rank by the click rate. - -In this example, we use the first method. - -We use the Kaggle `Click-through rate prediction` task \[[2](https://www.kaggle.com/c/avazu-ctr-prediction/data)\]. - -Please see the [data process](./dataset.md) for pre-processing data. - -The input data format for the demo model in this tutorial is as follows: - -``` -# \t \t click -1 23 190 \t 230:0.12 3421:0.9 23451:0.12 \t 0 -23 231 \t 1230:0.12 13421:0.9 \t 1 -``` - -Description： - -- `dnn input ids` one-hot coding. -- `lr input sparse values` Use `ID:VALUE` , values are preferaly scaled to the range `[-1, 1]`。 - -此外，模型训练时需要传入一个文件描述 dnn 和 lr两个子模型的输入维度，文件的格式如下： - -``` -dnn_input_dim: -lr_input_dim: -``` - - represents an integer value. - -`avazu_data_processor.py` can be used to download the data set \[[2](#参考文档)\]and pre-process the data. - -``` -usage: avazu_data_processer.py [-h] --data_path DATA_PATH --output_dir - OUTPUT_DIR - [--num_lines_to_detect NUM_LINES_TO_DETECT] - [--test_set_size TEST_SET_SIZE] - [--train_size TRAIN_SIZE] - -PaddlePaddle CTR example - -optional arguments: - -h, --help show this help message and exit - --data_path DATA_PATH - path of the Avazu dataset - --output_dir OUTPUT_DIR - directory to output - --num_lines_to_detect NUM_LINES_TO_DETECT - number of records to detect dataset's meta info - --test_set_size TEST_SET_SIZE - size of the validation dataset(default: 10000) - --train_size TRAIN_SIZE - size of the trainset (default: 100000) -``` - -- `data_path` The data path to be processed -- `output_dir` The output path of the data -- `num_lines_to_detect` The number of generated IDs -- `test_set_size` The number of rows for the test set -- `train_size` The number of rows of training set - -## Wide & Deep Learning Model - -Google proposed a model framework for Wide & Deep Learning to integrate the advantages of both DNNs suitable for learning abstract features and LR models for large sparse features. - - -### Introduction to the model - -Wide & Deep Learning Model\[[3](#References)\] is a relatively mature model, but this model is still being used in the CTR predicting task. Here we demonstrate the use of this model to complete the CTR predicting task. - -The model structure is as follows: - -

-
-Figure 2. Wide & Deep Model -

- -The wide part of the top side of the model can accommodate large-scale coefficient features and has some memory for some specific information (such as ID); and the Deep part of the bottom side of the model can learn the implicit relationship between features. - - -### Model Input - -The model has three inputs as follows. - -- `dnn_input` ，the Deep part of the input -- `lr_input` ，the wide part of the input -- `click` ， click on or not - -```python -dnn_merged_input = layer.data( - name='dnn_input', - type=paddle.data_type.sparse_binary_vector(self.dnn_input_dim)) - -lr_merged_input = layer.data( - name='lr_input', - type=paddle.data_type.sparse_vector(self.lr_input_dim)) - -click = paddle.layer.data(name='click', type=dtype.dense_vector(1)) -``` - -### Wide part - -Wide part uses of the LR model, but the activation function changed to `RELU` for speed. - -```python -def build_lr_submodel(): - fc = layer.fc( - input=lr_merged_input, size=1, name='lr', act=paddle.activation.Relu()) - return fc -``` - -### Deep part - -The Deep part uses a standard multi-layer DNN. - -```python -def build_dnn_submodel(dnn_layer_dims): - dnn_embedding = layer.fc(input=dnn_merged_input, size=dnn_layer_dims[0]) - _input_layer = dnn_embedding - for i, dim in enumerate(dnn_layer_dims[1:]): - fc = layer.fc( - input=_input_layer, - size=dim, - act=paddle.activation.Relu(), - name='dnn-fc-%d' % i) - _input_layer = fc - return _input_layer -``` - -### Combine - -The output section uses `sigmoid` function to output (0,1) as the prediction value. - -```python -# conbine DNN and LR submodels -def combine_submodels(dnn, lr): - merge_layer = layer.concat(input=[dnn, lr]) - fc = layer.fc( - input=merge_layer, - size=1, - name='output', - # use sigmoid function to approximate ctr, wihch is a float value between 0 and 1. - act=paddle.activation.Sigmoid()) - return fc -``` - -### Training -```python -dnn = build_dnn_submodel(dnn_layer_dims) -lr = build_lr_submodel() -output = combine_submodels(dnn, lr) - -# ============================================================================== -# cost and train period -# ============================================================================== -classification_cost = paddle.layer.multi_binary_label_cross_entropy_cost( - input=output, label=click) - - -paddle.init(use_gpu=False, trainer_count=11) - -params = paddle.parameters.create(classification_cost) - -optimizer = paddle.optimizer.Momentum(momentum=0) - -trainer = paddle.trainer.SGD( - cost=classification_cost, parameters=params, update_equation=optimizer) - -dataset = AvazuDataset(train_data_path, n_records_as_test=test_set_size) - -def event_handler(event): - if isinstance(event, paddle.event.EndIteration): - if event.batch_id % 100 == 0: - logging.warning("Pass %d, Samples %d, Cost %f" % ( - event.pass_id, event.batch_id * batch_size, event.cost)) - - if event.batch_id % 1000 == 0: - result = trainer.test( - reader=paddle.batch(dataset.test, batch_size=1000), - feeding=field_index) - logging.warning("Test %d-%d, Cost %f" % (event.pass_id, event.batch_id, - result.cost)) - - -trainer.train( - reader=paddle.batch( - paddle.reader.shuffle(dataset.train, buf_size=500), - batch_size=batch_size), - feeding=field_index, - event_handler=event_handler, - num_passes=100) -``` - -## Run training and testing -The model go through the following steps: - -1. Prepare training data - 1. Download train.gz from [Kaggle CTR](https://www.kaggle.com/c/avazu-ctr-prediction/data) . - 2. Unzip train.gz to get train.txt - 3. `mkdir -p output; python avazu_data_processer.py --data_path train.txt --output_dir output --num_lines_to_detect 1000 --test_set_size 100` 生成演示数据 -2. Execute `python train.py --train_data_path ./output/train.txt --test_data_path ./output/test.txt --data_meta_file ./output/data.meta.txt --model_type=0`. Start training. - -The argument options for `train.py` are as follows. - -``` -usage: train.py [-h] --train_data_path TRAIN_DATA_PATH - [--test_data_path TEST_DATA_PATH] [--batch_size BATCH_SIZE] - [--num_passes NUM_PASSES] - [--model_output_prefix MODEL_OUTPUT_PREFIX] --data_meta_file - DATA_META_FILE --model_type MODEL_TYPE - -PaddlePaddle CTR example - -optional arguments: - -h, --help show this help message and exit - --train_data_path TRAIN_DATA_PATH - path of training dataset - --test_data_path TEST_DATA_PATH - path of testing dataset - --batch_size BATCH_SIZE - size of mini-batch (default:10000) - --num_passes NUM_PASSES - number of passes to train - --model_output_prefix MODEL_OUTPUT_PREFIX - prefix of path for model to store (default: - ./ctr_models) - --data_meta_file DATA_META_FILE - path of data meta info file - --model_type MODEL_TYPE - model type, classification: 0, regression 1 (default - classification) -``` - -- `train_data_path` ： The path of the training set -- `test_data_path` : The path of the testing set -- `num_passes`: number of rounds of model training -- `data_meta_file`: Please refer to [数据和任务抽象](### 数据和任务抽象)的描述。 -- `model_type`: Model classification or regressio - - -## Use the training model for prediction -The training model can be used to predict new data, and the format of the forecast data is as follows. - - -``` -# \t -1 23 190 \t 230:0.12 3421:0.9 23451:0.12 -23 231 \t 1230:0.12 13421:0.9 -``` - -Here the only difference to the training data is that there is no label (i.e. `click` values). - -We now can use `infer.py` to perform inference. - -``` -usage: infer.py [-h] --model_gz_path MODEL_GZ_PATH --data_path DATA_PATH - --prediction_output_path PREDICTION_OUTPUT_PATH - [--data_meta_path DATA_META_PATH] --model_type MODEL_TYPE - -PaddlePaddle CTR example - -optional arguments: - -h, --help show this help message and exit - --model_gz_path MODEL_GZ_PATH - path of model parameters gz file - --data_path DATA_PATH - path of the dataset to infer - --prediction_output_path PREDICTION_OUTPUT_PATH - path to output the prediction - --data_meta_path DATA_META_PATH - path of trainset's meta info, default is ./data.meta - --model_type MODEL_TYPE - model type, classification: 0, regression 1 (default - classification) -``` - -- `model_gz_path_model`：path for `gz` compressed data. -- `data_path` ： -- `prediction_output_patj`：path for the predicted values s -- `data_meta_file` ：Please refer to [数据和任务抽象](### 数据和任务抽象)。 -- `model_type` ：Classification or regression - -The sample data can be predicted with the following command - -``` -python infer.py --model_gz_path --data_path output/infer.txt --prediction_output_path predictions.txt --data_meta_path data.meta.txt -``` - -The final prediction is written in `predictions.txt`。 - -## References -1. -2. -3. Cheng H T, Koc L, Harmsen J, et al. [Wide & deep learning for recommender systems](https://arxiv.org/pdf/1606.07792.pdf)[C]//Proceedings of the 1st Workshop on Deep Learning for Recommender Systems. ACM, 2016: 7-10. diff --git a/legacy/ctr/avazu_data_processer.py b/legacy/ctr/avazu_data_processer.py deleted file mode 100644 index dd3c1441f8f8b26473d15889198abb3593edfa51..0000000000000000000000000000000000000000 --- a/legacy/ctr/avazu_data_processer.py +++ /dev/null @@ -1,414 +0,0 @@ -import sys -import csv -import cPickle -import argparse -import os -import numpy as np - -from utils import logger, TaskMode - -parser = argparse.ArgumentParser(description="PaddlePaddle CTR example") -parser.add_argument( - '--data_path', type=str, required=True, help="path of the Avazu dataset") -parser.add_argument( - '--output_dir', type=str, required=True, help="directory to output") -parser.add_argument( - '--num_lines_to_detect', - type=int, - default=500000, - help="number of records to detect dataset's meta info") -parser.add_argument( - '--test_set_size', - type=int, - default=10000, - help="size of the validation dataset(default: 10000)") -parser.add_argument( - '--train_size', - type=int, - default=100000, - help="size of the trainset (default: 100000)") -args = parser.parse_args() -''' -The fields of the dataset are: - - 0. id: ad identifier - 1. click: 0/1 for non-click/click - 2. hour: format is YYMMDDHH, so 14091123 means 23:00 on Sept. 11, 2014 UTC. - 3. C1 -- anonymized categorical variable - 4. banner_pos - 5. site_id - 6. site_domain - 7. site_category - 8. app_id - 9. app_domain - 10. app_category - 11. device_id - 12. device_ip - 13. device_model - 14. device_type - 15. device_conn_type - 16. C14-C21 -- anonymized categorical variables - -We will treat the following fields as categorical features: - - - C1 - - banner_pos - - site_category - - app_category - - device_type - - device_conn_type - -and some other features as id features: - - - id - - site_id - - app_id - - device_id - -The `hour` field will be treated as a continuous feature and will be transformed -to one-hot representation which has 24 bits. - -This script will output 3 files: - -1. train.txt -2. test.txt -3. infer.txt - -all the files are for demo. -''' - -feature_dims = {} - -categorial_features = ( - 'C1 banner_pos site_category app_category ' + 'device_type device_conn_type' -).split() - -id_features = 'id site_id app_id device_id _device_id_cross_site_id'.split() - - -def get_all_field_names(mode=0): - ''' - @mode: int - 0 for train, 1 for test - @return: list of str - ''' - return categorial_features + ['hour'] + id_features + ['click'] \ - if mode == 0 else [] - - -class CategoryFeatureGenerator(object): - ''' - Generator category features. - - Register all records by calling `register` first, then call `gen` to generate - one-hot representation for a record. - ''' - - def __init__(self): - self.dic = {'unk': 0} - self.counter = 1 - - def register(self, key): - ''' - Register record. - ''' - if key not in self.dic: - self.dic[key] = self.counter - self.counter += 1 - - def size(self): - return len(self.dic) - - def gen(self, key): - ''' - Generate one-hot representation for a record. - ''' - if key not in self.dic: - res = self.dic['unk'] - else: - res = self.dic[key] - return [res] - - def __repr__(self): - return '' % len(self.dic) - - -class IDfeatureGenerator(object): - def __init__(self, max_dim, cross_fea0=None, cross_fea1=None): - ''' - @max_dim: int - Size of the id elements' space - ''' - self.max_dim = max_dim - self.cross_fea0 = cross_fea0 - self.cross_fea1 = cross_fea1 - - def gen(self, key): - ''' - Generate one-hot representation for records - ''' - return [hash(key) % self.max_dim] - - def gen_cross_fea(self, fea1, fea2): - key = str(fea1) + str(fea2) - return self.gen(key) - - def size(self): - return self.max_dim - - -class ContinuousFeatureGenerator(object): - def __init__(self, n_intervals): - self.min = sys.maxint - self.max = sys.minint - self.n_intervals = n_intervals - - def register(self, val): - self.min = min(self.minint, val) - self.max = max(self.maxint, val) - - def gen(self, val): - self.len_part = (self.max - self.min) / self.n_intervals - return (val - self.min) / self.len_part - - -# init all feature generators -fields = {} -for key in categorial_features: - fields[key] = CategoryFeatureGenerator() -for key in id_features: - # for cross features - if 'cross' in key: - feas = key[1:].split('_cross_') - fields[key] = IDfeatureGenerator(10000000, *feas) - # for normal ID features - else: - fields[key] = IDfeatureGenerator(10000) - -# used as feed_dict in PaddlePaddle -field_index = dict((key, id) - for id, key in enumerate(['dnn_input', 'lr_input', 'click'])) - - -def detect_dataset(path, topn, id_fea_space=10000): - ''' - Parse the first `topn` records to collect meta information of this dataset. - - NOTE the records should be randomly shuffled first. - ''' - # create categorical statis objects. - logger.warning('detecting dataset') - - with open(path, 'rb') as csvfile: - reader = csv.DictReader(csvfile) - for row_id, row in enumerate(reader): - if row_id > topn: - break - - for key in categorial_features: - fields[key].register(row[key]) - - for key, item in fields.items(): - feature_dims[key] = item.size() - - feature_dims['hour'] = 24 - feature_dims['click'] = 1 - - feature_dims['dnn_input'] = np.sum( - feature_dims[key] for key in categorial_features + ['hour']) + 1 - feature_dims['lr_input'] = np.sum(feature_dims[key] - for key in id_features) + 1 - return feature_dims - - -def load_data_meta(meta_path): - ''' - Load dataset's meta infomation. - ''' - feature_dims, fields = cPickle.load(open(meta_path, 'rb')) - return feature_dims, fields - - -def concat_sparse_vectors(inputs, dims): - ''' - Concaterate more than one sparse vectors into one. - - @inputs: list - list of sparse vector - @dims: list of int - dimention of each sparse vector - ''' - res = [] - assert len(inputs) == len(dims) - start = 0 - for no, vec in enumerate(inputs): - for v in vec: - res.append(v + start) - start += dims[no] - return res - - -class AvazuDataset(object): - ''' - Load AVAZU dataset as train set. - ''' - - def __init__(self, - train_path, - n_records_as_test=-1, - fields=None, - feature_dims=None): - self.train_path = train_path - self.n_records_as_test = n_records_as_test - self.fields = fields - # default is train mode. - self.mode = TaskMode.create_train() - - self.categorial_dims = [ - feature_dims[key] for key in categorial_features + ['hour'] - ] - self.id_dims = [feature_dims[key] for key in id_features] - - def train(self): - ''' - Load trainset. - ''' - logger.info("load trainset from %s" % self.train_path) - self.mode = TaskMode.create_train() - with open(self.train_path) as f: - reader = csv.DictReader(f) - - for row_id, row in enumerate(reader): - # skip top n lines - if self.n_records_as_test > 0 and row_id < self.n_records_as_test: - continue - - rcd = self._parse_record(row) - if rcd: - yield rcd - - def test(self): - ''' - Load testset. - ''' - logger.info("load testset from %s" % self.train_path) - self.mode = TaskMode.create_test() - with open(self.train_path) as f: - reader = csv.DictReader(f) - - for row_id, row in enumerate(reader): - # skip top n lines - if self.n_records_as_test > 0 and row_id > self.n_records_as_test: - break - - rcd = self._parse_record(row) - if rcd: - yield rcd - - def infer(self): - ''' - Load inferset. - ''' - logger.info("load inferset from %s" % self.train_path) - self.mode = TaskMode.create_infer() - with open(self.train_path) as f: - reader = csv.DictReader(f) - - for row_id, row in enumerate(reader): - rcd = self._parse_record(row) - if rcd: - yield rcd - - def _parse_record(self, row): - ''' - Parse a CSV row and get a record. - ''' - record = [] - for key in categorial_features: - record.append(self.fields[key].gen(row[key])) - record.append([int(row['hour'][-2:])]) - dense_input = concat_sparse_vectors(record, self.categorial_dims) - - record = [] - for key in id_features: - if 'cross' not in key: - record.append(self.fields[key].gen(row[key])) - else: - fea0 = self.fields[key].cross_fea0 - fea1 = self.fields[key].cross_fea1 - record.append(self.fields[key].gen_cross_fea(row[fea0], row[ - fea1])) - - sparse_input = concat_sparse_vectors(record, self.id_dims) - - record = [dense_input, sparse_input] - - if not self.mode.is_infer(): - record.append(list((int(row['click']), ))) - return record - - -def ids2dense(vec, dim): - return vec - - -def ids2sparse(vec): - return ["%d:1" % x for x in vec] - - -detect_dataset(args.data_path, args.num_lines_to_detect) -dataset = AvazuDataset( - args.data_path, - args.test_set_size, - fields=fields, - feature_dims=feature_dims) - -output_trainset_path = os.path.join(args.output_dir, 'train.txt') -output_testset_path = os.path.join(args.output_dir, 'test.txt') -output_infer_path = os.path.join(args.output_dir, 'infer.txt') -output_meta_path = os.path.join(args.output_dir, 'data.meta.txt') - -with open(output_trainset_path, 'w') as f: - for id, record in enumerate(dataset.train()): - if id and id % 10000 == 0: - logger.info("load %d records" % id) - if id > args.train_size: - break - dnn_input, lr_input, click = record - dnn_input = ids2dense(dnn_input, feature_dims['dnn_input']) - lr_input = ids2sparse(lr_input) - line = "%s\t%s\t%d\n" % (' '.join(map(str, dnn_input)), - ' '.join(map(str, lr_input)), click[0]) - f.write(line) - logger.info('write to %s' % output_trainset_path) - -with open(output_testset_path, 'w') as f: - for id, record in enumerate(dataset.test()): - dnn_input, lr_input, click = record - dnn_input = ids2dense(dnn_input, feature_dims['dnn_input']) - lr_input = ids2sparse(lr_input) - line = "%s\t%s\t%d\n" % (' '.join(map(str, dnn_input)), - ' '.join(map(str, lr_input)), click[0]) - f.write(line) - logger.info('write to %s' % output_testset_path) - -with open(output_infer_path, 'w') as f: - for id, record in enumerate(dataset.infer()): - dnn_input, lr_input = record - dnn_input = ids2dense(dnn_input, feature_dims['dnn_input']) - lr_input = ids2sparse(lr_input) - line = "%s\t%s\n" % ( - ' '.join(map(str, dnn_input)), - ' '.join(map(str, lr_input)), ) - f.write(line) - if id > args.test_set_size: - break - logger.info('write to %s' % output_infer_path) - -with open(output_meta_path, 'w') as f: - lines = [ - "dnn_input_dim: %d" % feature_dims['dnn_input'], - "lr_input_dim: %d" % feature_dims['lr_input'] - ] - f.write('\n'.join(lines)) - logger.info('write data meta into %s' % output_meta_path) diff --git a/legacy/ctr/dataset.md b/legacy/ctr/dataset.md deleted file mode 100644 index 16c0f9784bf3409ac5bbe704f932a9b28680fbf8..0000000000000000000000000000000000000000 --- a/legacy/ctr/dataset.md +++ /dev/null @@ -1,296 +0,0 @@ -# 数据及处理 -## 数据集介绍 - -本教程演示使用Kaggle上CTR任务的数据集\[[3](#参考文献)\]的预处理方法，最终产生本模型需要的格式，详细的数据格式参考[README.md](./README.md)。 - -Wide && Deep Model\[[2](#参考文献)\]的优势是融合稠密特征和大规模稀疏特征， -因此特征处理方面也针对稠密和稀疏两种特征作处理， -其中Deep部分的稠密值全部转化为ID类特征， -通过embedding 来转化为稠密的向量输入；Wide部分主要通过ID的叉乘提升维度。 - -数据集使用 `csv` 格式存储，其中各个字段内容如下： - -- `id` : ad identifier -- `click` : 0/1 for non-click/click -- `hour` : format is YYMMDDHH, so 14091123 means 23:00 on Sept. 11, 2014 UTC. -- `C1` : anonymized categorical variable -- `banner_pos` -- `site_id` -- `site_domain` -- `site_category` -- `app_id` -- `app_domain` -- `app_category` -- `device_id` -- `device_ip` -- `device_model` -- `device_type` -- `device_conn_type` -- `C14-C21` : anonymized categorical variables - - -## 特征提取 - -下面我们会简单演示几种特征的提取方式。 - -原始数据中的特征可以分为以下几类： - -1. ID 类特征（稀疏，数量多） -- `id` -- `site_id` -- `app_id` -- `device_id` - -2. 类别类特征（稀疏，但数量有限） - -- `C1` -- `site_category` -- `device_type` -- `C14-C21` - -3. 数值型特征转化为类别型特征 - -- hour (可以转化成数值，也可以按小时为单位转化为类别） - -### 类别类特征 - -类别类特征的提取方法有以下两种： - -1. One-hot 表示作为特征 -2. 类似词向量，用一个 Embedding 将每个类别映射到对应的向量 - - -### ID 类特征 - -ID 类特征的特点是稀疏数据，但量比较大，直接使用 One-hot 表示时维度过大。 - -一般会作如下处理： - -1. 确定表示的最大维度 N -2. newid = id % N -3. 用 newid 作为类别类特征使用 - -上面的方法尽管存在一定的碰撞概率，但能够处理任意数量的 ID 特征，并保留一定的效果\[[2](#参考文献)\]。 - -### 数值型特征 - -一般会做如下处理： - -- 归一化，直接作为特征输入模型 -- 用区间分割处理成类别类特征，稀疏化表示，模糊细微上的差别 - -## 特征处理 - - -### 类别型特征 - -类别型特征有有限多种值，在模型中，我们一般使用 Embedding将每种值映射为连续值的向量。 - -这种特征在输入到模型时，一般使用 One-hot 表示，相关处理方法如下： - -```python -class CategoryFeatureGenerator(object): - ''' - Generator category features. - - Register all records by calling ~register~ first, then call ~gen~ to generate - one-hot representation for a record. - ''' - - def __init__(self): - self.dic = {'unk': 0} - self.counter = 1 - - def register(self, key): - ''' - Register record. - ''' - if key not in self.dic: - self.dic[key] = self.counter - self.counter += 1 - - def size(self): - return len(self.dic) - - def gen(self, key): - ''' - Generate one-hot representation for a record. - ''' - if key not in self.dic: - res = self.dic['unk'] - else: - res = self.dic[key] - return [res] - - def __repr__(self): - return '' % len(self.dic) -``` - -`CategoryFeatureGenerator` 需要先扫描数据集，得到该类别对应的项集合，之后才能开始生成特征。 - -我们的实验数据集\[[3](https://www.kaggle.com/c/avazu-ctr-prediction/data)\]已经经过shuffle，可以扫描前面一定数目的记录来近似总的类别项集合（等价于随机抽样）， -对于没有抽样上的低频类别项，可以用一个 UNK 的特殊值表示。 - -```python -fields = {} -for key in categorial_features: - fields[key] = CategoryFeatureGenerator() - -def detect_dataset(path, topn, id_fea_space=10000): - ''' - Parse the first `topn` records to collect meta information of this dataset. - - NOTE the records should be randomly shuffled first. - ''' - # create categorical statis objects. - - with open(path, 'rb') as csvfile: - reader = csv.DictReader(csvfile) - for row_id, row in enumerate(reader): - if row_id > topn: - break - - for key in categorial_features: - fields[key].register(row[key]) -``` - -`CategoryFeatureGenerator` 在注册得到数据集中对应类别信息后，可以对相应记录生成对应的特征表示： - -```python -record = [] -for key in categorial_features: - record.append(fields[key].gen(row[key])) -``` - -本任务中，类别类特征会输入到 DNN 中使用。 - -### ID 类特征 - -ID 类特征代稀疏值，且值的空间很大的情况，一般用模操作规约到一个有限空间， -之后可以当成类别类特征使用，这里我们会将 ID 类特征输入到 LR 模型中使用。 - -```python -class IDfeatureGenerator(object): - def __init__(self, max_dim): - ''' - @max_dim: int - Size of the id elements' space - ''' - self.max_dim = max_dim - - def gen(self, key): - ''' - Generate one-hot representation for records - ''' - return [hash(key) % self.max_dim] - - def size(self): - return self.max_dim -``` - -`IDfeatureGenerator` 不需要预先初始化，可以直接生成特征，比如 - -```python -record = [] -for key in id_features: - if 'cross' not in key: - record.append(fields[key].gen(row[key])) -``` - -### 交叉类特征 - -LR 模型作为 Wide & Deep model 的 `wide` 部分，可以输入很 wide 的数据（特征空间的维度很大）， -为了充分利用这个优势，我们将演示交叉组合特征构建成更大维度特征的情况，之后塞入到模型中训练。 - -这里我们依旧使用模操作来约束最终组合出的特征空间的大小，具体实现是直接在 `IDfeatureGenerator` 中添加一个 `gen_cross_feature` 的方法： - -```python -def gen_cross_fea(self, fea1, fea2): - key = str(fea1) + str(fea2) - return self.gen(key) -``` - -比如，我们觉得原始数据中， `device_id` 和 `site_id` 有一些关联（比如某个 device 倾向于浏览特定 site)， -我们通过组合出两者组合来捕捉这类信息。 - -```python -fea0 = fields[key].cross_fea0 -fea1 = fields[key].cross_fea1 -record.append( - fields[key].gen_cross_fea(row[fea0], row[fea1])) -``` - -### 特征维度 -#### Deep submodel(DNN)特征 -| feature | dimention | -|------------------|-----------| -| app_category | 21 | -| site_category | 22 | -| device_conn_type | 5 | -| hour | 24 | -| banner_pos | 7 | -| **Total** | 79 | - -#### Wide submodel(LR)特征 -| Feature | Dimention | -|---------------------|-----------| -| id | 10000 | -| site_id | 10000 | -| app_id | 10000 | -| device_id | 10000 | -| device_id X site_id | 1000000 | -| **Total** | 1,040,000 | - -## 输入到 PaddlePaddle 中 - -Deep 和 Wide 两部分均以 `sparse_binary_vector` 的格式 \[[1](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/api/v1/data_provider/pydataprovider2_en.rst)\] 输入，输入前需要将相关特征拼合，模型最终只接受 3 个 input， -分别是 - -1. `dnn input` ，DNN 的输入 -2. `lr input` , LR 的输入 -3. `click` ， 标签 - -拼合特征的方法： - -```python -def concat_sparse_vectors(inputs, dims): - ''' - concaterate sparse vectors into one - - @inputs: list - list of sparse vector - @dims: list of int - dimention of each sparse vector - ''' - res = [] - assert len(inputs) == len(dims) - start = 0 - for no, vec in enumerate(inputs): - for v in vec: - res.append(v + start) - start += dims[no] - return res -``` - -生成最终特征的代码如下： - -```python -# dimentions of the features -categorial_dims = [ - feature_dims[key] for key in categorial_features + ['hour'] -] -id_dims = [feature_dims[key] for key in id_features] - -dense_input = concat_sparse_vectors(record, categorial_dims) -sparse_input = concat_sparse_vectors(record, id_dims) - -record = [dense_input, sparse_input] -record.append(list((int(row['click']), ))) -yield record -``` - -## 参考文献 - -1. -2. Mikolov T, Deoras A, Povey D, et al. [Strategies for training large scale neural network language models](https://www.researchgate.net/profile/Lukas_Burget/publication/241637478_Strategies_for_training_large_scale_neural_network_language_models/links/542c14960cf27e39fa922ed3.pdf)[C]//Automatic Speech Recognition and Understanding (ASRU), 2011 IEEE Workshop on. IEEE, 2011: 196-201. -3. diff --git a/legacy/ctr/images/lr_vs_dnn.jpg b/legacy/ctr/images/lr_vs_dnn.jpg deleted file mode 100644 index 50a0db583cd9b6e1a5bc0f83a28ab6e22d649931..0000000000000000000000000000000000000000 Binary files a/legacy/ctr/images/lr_vs_dnn.jpg and /dev/null differ diff --git a/legacy/ctr/images/wide_deep.png b/legacy/ctr/images/wide_deep.png deleted file mode 100644 index 616f88cb22607c1c6bcbe4312644f632ef284e8e..0000000000000000000000000000000000000000 Binary files a/legacy/ctr/images/wide_deep.png and /dev/null differ diff --git a/legacy/ctr/infer.py b/legacy/ctr/infer.py deleted file mode 100644 index 6541c74638df63a9304989c2ccaff0ff4c00463a..0000000000000000000000000000000000000000 --- a/legacy/ctr/infer.py +++ /dev/null @@ -1,79 +0,0 @@ -import gzip -import argparse -import itertools - -import paddle.v2 as paddle -import network_conf -from train import dnn_layer_dims -import reader -from utils import logger, ModelType - -parser = argparse.ArgumentParser(description="PaddlePaddle CTR example") -parser.add_argument( - '--model_gz_path', - type=str, - required=True, - help="path of model parameters gz file") -parser.add_argument( - '--data_path', type=str, required=True, help="path of the dataset to infer") -parser.add_argument( - '--prediction_output_path', - type=str, - required=True, - help="path to output the prediction") -parser.add_argument( - '--data_meta_path', - type=str, - default="./data.meta", - help="path of trainset's meta info, default is ./data.meta") -parser.add_argument( - '--model_type', - type=int, - required=True, - default=ModelType.CLASSIFICATION, - help='model type, classification: %d, regression %d (default classification)' - % (ModelType.CLASSIFICATION, ModelType.REGRESSION)) - -args = parser.parse_args() - -paddle.init(use_gpu=False, trainer_count=1) - - -class CTRInferer(object): - def __init__(self, param_path): - logger.info("create CTR model") - dnn_input_dim, lr_input_dim = reader.load_data_meta(args.data_meta_path) - # create the mdoel - self.ctr_model = network_conf.CTRmodel( - dnn_layer_dims, - dnn_input_dim, - lr_input_dim, - model_type=ModelType(args.model_type), - is_infer=True) - # load parameter - logger.info("load model parameters from %s" % param_path) - self.parameters = paddle.parameters.Parameters.from_tar( - gzip.open(param_path, 'r')) - self.inferer = paddle.inference.Inference( - output_layer=self.ctr_model.model, - parameters=self.parameters, ) - - def infer(self, data_path): - logger.info("infer data...") - dataset = reader.Dataset() - infer_reader = paddle.batch( - dataset.infer(args.data_path), batch_size=1000) - logger.warning('write predictions to %s' % args.prediction_output_path) - output_f = open(args.prediction_output_path, 'w') - for id, batch in enumerate(infer_reader()): - res = self.inferer.infer(input=batch) - predictions = [x for x in itertools.chain.from_iterable(res)] - assert len(batch) == len( - predictions), "predict error, %d inputs, but %d predictions" % ( - len(batch), len(predictions)) - output_f.write('\n'.join(map(str, predictions)) + '\n') - - -if __name__ == '__main__': - ctr_inferer = CTRInferer(args.model_gz_path) - ctr_inferer.infer(args.data_path) diff --git a/legacy/ctr/network_conf.py b/legacy/ctr/network_conf.py deleted file mode 100644 index bcff49ee05e1d8cc80e2fdd28a771bf9bf9502e3..0000000000000000000000000000000000000000 --- a/legacy/ctr/network_conf.py +++ /dev/null @@ -1,104 +0,0 @@ -import paddle.v2 as paddle -from paddle.v2 import layer -from paddle.v2 import data_type as dtype -from utils import logger, ModelType - - -class CTRmodel(object): - ''' - A CTR model which implements wide && deep learning model. - ''' - - def __init__(self, - dnn_layer_dims, - dnn_input_dim, - lr_input_dim, - model_type=ModelType.create_classification(), - is_infer=False): - ''' - @dnn_layer_dims: list of integer - dims of each layer in dnn - @dnn_input_dim: int - size of dnn's input layer - @lr_input_dim: int - size of lr's input layer - @is_infer: bool - whether to build a infer model - ''' - self.dnn_layer_dims = dnn_layer_dims - self.dnn_input_dim = dnn_input_dim - self.lr_input_dim = lr_input_dim - self.model_type = model_type - self.is_infer = is_infer - - self._declare_input_layers() - - self.dnn = self._build_dnn_submodel_(self.dnn_layer_dims) - self.lr = self._build_lr_submodel_() - - # model's prediction - # TODO(superjom) rename it to prediction - if self.model_type.is_classification(): - self.model = self._build_classification_model(self.dnn, self.lr) - if self.model_type.is_regression(): - self.model = self._build_regression_model(self.dnn, self.lr) - - def _declare_input_layers(self): - self.dnn_merged_input = layer.data( - name='dnn_input', - type=paddle.data_type.sparse_binary_vector(self.dnn_input_dim)) - - self.lr_merged_input = layer.data( - name='lr_input', - type=paddle.data_type.sparse_float_vector(self.lr_input_dim)) - - if not self.is_infer: - self.click = paddle.layer.data( - name='click', type=dtype.dense_vector(1)) - - def _build_dnn_submodel_(self, dnn_layer_dims): - ''' - build DNN submodel. - ''' - dnn_embedding = layer.fc(input=self.dnn_merged_input, - size=dnn_layer_dims[0]) - _input_layer = dnn_embedding - for i, dim in enumerate(dnn_layer_dims[1:]): - fc = layer.fc(input=_input_layer, - size=dim, - act=paddle.activation.Relu(), - name='dnn-fc-%d' % i) - _input_layer = fc - return _input_layer - - def _build_lr_submodel_(self): - ''' - config LR submodel - ''' - fc = layer.fc(input=self.lr_merged_input, - size=1, - act=paddle.activation.Relu()) - return fc - - def _build_classification_model(self, dnn, lr): - merge_layer = layer.concat(input=[dnn, lr]) - self.output = layer.fc( - input=merge_layer, - size=1, - # use sigmoid function to approximate ctr rate, a float value between 0 and 1. - act=paddle.activation.Sigmoid()) - - if not self.is_infer: - self.train_cost = paddle.layer.multi_binary_label_cross_entropy_cost( - input=self.output, label=self.click) - return self.output - - def _build_regression_model(self, dnn, lr): - merge_layer = layer.concat(input=[dnn, lr]) - self.output = layer.fc(input=merge_layer, - size=1, - act=paddle.activation.Sigmoid()) - if not self.is_infer: - self.train_cost = paddle.layer.square_error_cost( - input=self.output, label=self.click) - return self.output diff --git a/legacy/ctr/reader.py b/legacy/ctr/reader.py deleted file mode 100644 index cafa2349ed0e51a8de65dbeeea8b345edcf0a879..0000000000000000000000000000000000000000 --- a/legacy/ctr/reader.py +++ /dev/null @@ -1,64 +0,0 @@ -from utils import logger, TaskMode, load_dnn_input_record, load_lr_input_record - -feeding_index = {'dnn_input': 0, 'lr_input': 1, 'click': 2} - - -class Dataset(object): - def train(self, path): - ''' - Load trainset. - ''' - logger.info("load trainset from %s" % path) - mode = TaskMode.create_train() - return self._parse_creator(path, mode) - - def test(self, path): - ''' - Load testset. - ''' - logger.info("load testset from %s" % path) - mode = TaskMode.create_test() - return self._parse_creator(path, mode) - - def infer(self, path): - ''' - Load infer set. - ''' - logger.info("load inferset from %s" % path) - mode = TaskMode.create_infer() - return self._parse_creator(path, mode) - - def _parse_creator(self, path, mode): - ''' - Parse dataset. - ''' - - def _parse(): - with open(path) as f: - for line_id, line in enumerate(f): - fs = line.strip().split('\t') - dnn_input = load_dnn_input_record(fs[0]) - lr_input = load_lr_input_record(fs[1]) - if not mode.is_infer(): - click = [int(fs[2])] - yield dnn_input, lr_input, click - else: - yield dnn_input, lr_input - - return _parse - - -def load_data_meta(path): - ''' - load data meta info from path, return (dnn_input_dim, lr_input_dim) - ''' - with open(path) as f: - lines = f.read().split('\n') - err_info = "wrong meta format" - assert len(lines) == 2, err_info - assert 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[ - 1], err_info - res = map(int, [_.split(':')[1] for _ in lines]) - logger.info('dnn input dim: %d' % res[0]) - logger.info('lr input dim: %d' % res[1]) - return res diff --git a/legacy/ctr/train.py b/legacy/ctr/train.py deleted file mode 100644 index de7add61d65aba363cc17bed49d32c9054600108..0000000000000000000000000000000000000000 --- a/legacy/ctr/train.py +++ /dev/null @@ -1,112 +0,0 @@ -import argparse -import gzip - -import reader -import paddle.v2 as paddle -from utils import logger, ModelType -from network_conf import CTRmodel - - -def parse_args(): - parser = argparse.ArgumentParser(description="PaddlePaddle CTR example") - parser.add_argument( - '--train_data_path', - type=str, - required=True, - help="path of training dataset") - parser.add_argument( - '--test_data_path', type=str, help='path of testing dataset') - parser.add_argument( - '--batch_size', - type=int, - default=10000, - help="size of mini-batch (default:10000)") - parser.add_argument( - '--num_passes', type=int, default=10, help="number of passes to train") - parser.add_argument( - '--model_output_prefix', - type=str, - default='./ctr_models', - help='prefix of path for model to store (default: ./ctr_models)') - parser.add_argument( - '--data_meta_file', - type=str, - required=True, - help='path of data meta info file', ) - parser.add_argument( - '--model_type', - type=int, - required=True, - default=ModelType.CLASSIFICATION, - help='model type, classification: %d, regression %d (default classification)' - % (ModelType.CLASSIFICATION, ModelType.REGRESSION)) - - return parser.parse_args() - - -dnn_layer_dims = [128, 64, 32, 1] - -# ============================================================================== -# cost and train period -# ============================================================================== - - -def train(): - args = parse_args() - args.model_type = ModelType(args.model_type) - paddle.init(use_gpu=False, trainer_count=1) - dnn_input_dim, lr_input_dim = reader.load_data_meta(args.data_meta_file) - - # create ctr model. - model = CTRmodel( - dnn_layer_dims, - dnn_input_dim, - lr_input_dim, - model_type=args.model_type, - is_infer=False) - - params = paddle.parameters.create(model.train_cost) - optimizer = paddle.optimizer.AdaGrad() - - trainer = paddle.trainer.SGD(cost=model.train_cost, - parameters=params, - update_equation=optimizer) - - dataset = reader.Dataset() - - def __event_handler__(event): - if isinstance(event, paddle.event.EndIteration): - num_samples = event.batch_id * args.batch_size - if event.batch_id % 100 == 0: - logger.warning("Pass %d, Samples %d, Cost %f, %s" % ( - event.pass_id, num_samples, event.cost, event.metrics)) - - if event.batch_id % 1000 == 0: - if args.test_data_path: - result = trainer.test( - reader=paddle.batch( - dataset.test(args.test_data_path), - batch_size=args.batch_size), - feeding=reader.feeding_index) - logger.warning("Test %d-%d, Cost %f, %s" % - (event.pass_id, event.batch_id, result.cost, - result.metrics)) - - path = "{}-pass-{}-batch-{}-test-{}.tar.gz".format( - args.model_output_prefix, event.pass_id, event.batch_id, - result.cost) - with gzip.open(path, 'w') as f: - trainer.save_parameter_to_tar(f) - - trainer.train( - reader=paddle.batch( - paddle.reader.shuffle( - dataset.train(args.train_data_path), buf_size=500), - batch_size=args.batch_size), - feeding=reader.feeding_index, - event_handler=__event_handler__, - num_passes=args.num_passes) - - -if __name__ == '__main__': - train() diff --git a/legacy/ctr/utils.py b/legacy/ctr/utils.py deleted file mode 100644 index 437554c3c291d5a74cc0b3844c8684c73b189a19..0000000000000000000000000000000000000000 --- a/legacy/ctr/utils.py +++ /dev/null @@ -1,70 +0,0 @@ -import logging - -logging.basicConfig() -logger = logging.getLogger("paddle") -logger.setLevel(logging.INFO) - - -class TaskMode: - TRAIN_MODE = 0 - TEST_MODE = 1 - INFER_MODE = 2 - - def __init__(self, mode): - self.mode = mode - - def is_train(self): - return self.mode == self.TRAIN_MODE - - def is_test(self): - return self.mode == self.TEST_MODE - - def is_infer(self): - return self.mode == self.INFER_MODE - - @staticmethod - def create_train(): - return TaskMode(TaskMode.TRAIN_MODE) - - @staticmethod - def create_test(): - return TaskMode(TaskMode.TEST_MODE) - - @staticmethod - def create_infer(): - return TaskMode(TaskMode.INFER_MODE) - - -class ModelType: - CLASSIFICATION = 0 - REGRESSION = 1 - - def __init__(self, mode): - self.mode = mode - - def is_classification(self): - return self.mode == self.CLASSIFICATION - - def is_regression(self): - return self.mode == self.REGRESSION - - @staticmethod - def create_classification(): - return ModelType(ModelType.CLASSIFICATION) - - @staticmethod - def create_regression(): - return ModelType(ModelType.REGRESSION) - - -def load_dnn_input_record(sent): - return map(int, sent.split()) - - -def load_lr_input_record(sent): - res = [] - for _ in [x.split(':') for x in sent.split()]: - res.append(( - int(_[0]), - float(_[1]), )) - return res diff --git a/legacy/deep_fm/README.cn.md b/legacy/deep_fm/README.cn.md deleted file mode 100644 index 1f651acbde0078340dab06c551f583ca2b1dd86c..0000000000000000000000000000000000000000 --- a/legacy/deep_fm/README.cn.md +++ /dev/null @@ -1,76 +0,0 @@ -运行本目录下的程序示例需要使用PaddlePaddle v0.10.0 版本。如果您的PaddlePaddle安装版本低于此要求，请按照[安装文档](http://www.paddlepaddle.org/docs/develop/documentation/en/build_and_install/pip_install_en.html)中的说明更新PaddlePaddle安装版本。 - ---- - -# 基于深度因子分解机的点击率预估模型 - -## 介绍 -本模型实现了下述论文中提出的DeepFM模型： - -```text -@inproceedings{guo2017deepfm, - title={DeepFM: A Factorization-Machine based Neural Network for CTR Prediction}, - author={Huifeng Guo, Ruiming Tang, Yunming Ye, Zhenguo Li and Xiuqiang He}, - booktitle={the Twenty-Sixth International Joint Conference on Artificial Intelligence (IJCAI)}, - pages={1725--1731}, - year={2017} -} -``` - -DeepFM模型把因子分解机和深度神经网络的低阶和高阶特征的相互作用结合起来，有关因子分解机的详细信息，请参考论文[因子分解机](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf)。 - -## 数据集 -本文使用的是Kaggle公司举办的[展示广告竞赛](https://www.kaggle.com/c/criteo-display-ad-challenge/)中所使用的Criteo数据集。 - -每一行是一次广告展示的特征，第一列是一个标签，表示这次广告展示是否被点击。总共有39个特征，其中13个特征采用整型值，另外26个特征是类别类特征。测试集中是没有标签的。 - -下载数据集： -```bash -cd data && ./download.sh && cd .. -``` - -## 模型 -DeepFM模型是由因子分解机（FM）和深度神经网络（DNN）组成的。所有的输入特征都会同时输入FM和DNN，最后把FM和DNN的输出结合在一起形成最终的输出。DNN中稀疏特征生成的嵌入层与FM层中的隐含向量（因子）共享参数。 - -PaddlePaddle中的因子分解机层负责计算二阶组合特征的相互关系。以下的代码示例结合了因子分解机层和全连接层，形成了完整的的因子分解机： - -```python -def fm_layer(input, factor_size): - first_order = paddle.layer.fc(input=input, size=1, act=paddle.activation.Linear()) - second_order = paddle.layer.factorization_machine(input=input, factor_size=factor_size) - fm = paddle.layer.addto(input=[first_order, second_order], - act=paddle.activation.Linear(), - bias_attr=False) - return fm -``` - -## 数据准备 -处理原始数据集，整型特征使用min-max归一化方法规范到[0, 1]，类别类特征使用了one-hot编码。原始数据集分割成两部分：90%用于训练，其他10%用于训练过程中的验证。 - -```bash -python preprocess.py --datadir ./data/raw --outdir ./data -``` - -## 训练 -训练的命令行选项可以通过`python train.py -h`列出。 - -训练模型： -```bash -python train.py \ - --train_data_path data/train.txt \ - --test_data_path data/valid.txt \ - 2>&1 | tee train.log -``` - -训练到第9轮的第40000个batch后，测试的AUC为0.807178，误差（cost）为0.445196。 - -## 预测 -预测的命令行选项可以通过`python infer.py -h`列出。 - -对测试集进行预测： -```bash -python infer.py \ - --model_gz_path models/model-pass-9-batch-10000.tar.gz \ - --data_path data/test.txt \ - --prediction_output_path ./predict.txt -``` diff --git a/legacy/deep_fm/README.md b/legacy/deep_fm/README.md deleted file mode 100644 index 6e2c6fad38d2e9e9db8d17c4967196b4f1cc5a36..0000000000000000000000000000000000000000 --- a/legacy/deep_fm/README.md +++ /dev/null @@ -1,95 +0,0 @@ -The minimum PaddlePaddle version needed for the code sample in this directory is v0.11.0. If you are on a version of PaddlePaddle earlier than v0.11.0, [please update your installation](http://www.paddlepaddle.org/docs/develop/documentation/en/build_and_install/pip_install_en.html). - ---- - -# Deep Factorization Machine for Click-Through Rate prediction - -## Introduction -This model implements the DeepFM proposed in the following paper: - -```text -@inproceedings{guo2017deepfm, - title={DeepFM: A Factorization-Machine based Neural Network for CTR Prediction}, - author={Huifeng Guo, Ruiming Tang, Yunming Ye, Zhenguo Li and Xiuqiang He}, - booktitle={the Twenty-Sixth International Joint Conference on Artificial Intelligence (IJCAI)}, - pages={1725--1731}, - year={2017} -} -``` - -The DeepFm combines factorization machine and deep neural networks to model -both low order and high order feature interactions. For details of the -factorization machines, please refer to the paper [factorization -machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf) - -## Dataset -This example uses Criteo dataset which was used for the [Display Advertising -Challenge](https://www.kaggle.com/c/criteo-display-ad-challenge/) -hosted by Kaggle. - -Each row is the features for an ad display and the first column is a label -indicating whether this ad has been clicked or not. There are 39 features in -total. 13 features take integer values and the other 26 features are -categorical features. For the test dataset, the labels are omitted. - -Download dataset: -```bash -cd data && ./download.sh && cd .. -``` - -## Model -The DeepFM model is composed of the factorization machine layer (FM) and deep -neural networks (DNN). All the input features are feeded to both FM and DNN. -The output from FM and DNN are combined to form the final output. The embedding -layer for sparse features in the DNN shares the parameters with the latent -vectors (factors) of the FM layer. - -The factorization machine layer in PaddlePaddle computes the second order -interactions. The following code example combines the factorization machine -layer and fully connected layer to form the full version of factorization -machine: - -```python -def fm_layer(input, factor_size): - first_order = paddle.layer.fc(input=input, size=1, act=paddle.activation.Linear()) - second_order = paddle.layer.factorization_machine(input=input, factor_size=factor_size) - fm = paddle.layer.addto(input=[first_order, second_order], - act=paddle.activation.Linear(), - bias_attr=False) - return fm -``` - -## Data preparation -To preprocess the raw dataset, the integer features are clipped then min-max -normalized to [0, 1] and the categorical features are one-hot encoded. The raw -training dataset are splited such that 90% are used for training and the other -10% are used for validation during training. - -```bash -python preprocess.py --datadir ./data/raw --outdir ./data -``` - -## Train -The command line options for training can be listed by `python train.py -h`. - -To train the model: -```bash -python train.py \ - --train_data_path data/train.txt \ - --test_data_path data/valid.txt \ - 2>&1 | tee train.log -``` - -After training pass 9 batch 40000, the testing AUC is `0.807178` and the testing -cost is `0.445196`. - -## Infer -The command line options for infering can be listed by `python infer.py -h`. - -To make inference for the test dataset: -```bash -python infer.py \ - --model_gz_path models/model-pass-9-batch-10000.tar.gz \ - --data_path data/test.txt \ - --prediction_output_path ./predict.txt -``` diff --git a/legacy/deep_fm/data/download.sh b/legacy/deep_fm/data/download.sh deleted file mode 100755 index 466a22f2c6cc885cea0a1468f3043cb59c611b59..0000000000000000000000000000000000000000 --- a/legacy/deep_fm/data/download.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -wget --no-check-certificate https://s3-eu-west-1.amazonaws.com/criteo-labs/dac.tar.gz -tar zxf dac.tar.gz -rm -f dac.tar.gz - -mkdir raw -mv ./*.txt raw/ diff --git a/legacy/deep_fm/infer.py b/legacy/deep_fm/infer.py deleted file mode 100755 index 40a5929780090d403b8b905f8e949f1f8a020eb3..0000000000000000000000000000000000000000 --- a/legacy/deep_fm/infer.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import gzip -import argparse -import itertools - -import paddle.v2 as paddle - -from network_conf import DeepFM -import reader - - -def parse_args(): - parser = argparse.ArgumentParser(description="PaddlePaddle DeepFM example") - parser.add_argument( - '--model_gz_path', - type=str, - required=True, - help="The path of model parameters gz file") - parser.add_argument( - '--data_path', - type=str, - required=True, - help="The path of the dataset to infer") - parser.add_argument( - '--prediction_output_path', - type=str, - required=True, - help="The path to output the prediction") - parser.add_argument( - '--factor_size', - type=int, - default=10, - help="The factor size for the factorization machine (default:10)") - - return parser.parse_args() - - -def infer(): - args = parse_args() - - paddle.init(use_gpu=False, trainer_count=1) - - model = DeepFM(args.factor_size, infer=True) - - parameters = paddle.parameters.Parameters.from_tar( - gzip.open(args.model_gz_path, 'r')) - - inferer = paddle.inference.Inference( - output_layer=model, parameters=parameters) - - dataset = reader.Dataset() - - infer_reader = paddle.batch(dataset.infer(args.data_path), batch_size=1000) - - with open(args.prediction_output_path, 'w') as out: - for id, batch in enumerate(infer_reader()): - res = inferer.infer(input=batch) - predictions = [x for x in itertools.chain.from_iterable(res)] - out.write('\n'.join(map(str, predictions)) + '\n') - - -if __name__ == '__main__': - infer() diff --git a/legacy/deep_fm/network_conf.py b/legacy/deep_fm/network_conf.py deleted file mode 100644 index 545fe07b8197e3379eb5a6f34c3134b813a4684e..0000000000000000000000000000000000000000 --- a/legacy/deep_fm/network_conf.py +++ /dev/null @@ -1,75 +0,0 @@ -import paddle.v2 as paddle - -dense_feature_dim = 13 -sparse_feature_dim = 117568 - - -def fm_layer(input, factor_size, fm_param_attr): - first_order = paddle.layer.fc(input=input, - size=1, - act=paddle.activation.Linear()) - second_order = paddle.layer.factorization_machine( - input=input, - factor_size=factor_size, - act=paddle.activation.Linear(), - param_attr=fm_param_attr) - out = paddle.layer.addto( - input=[first_order, second_order], - act=paddle.activation.Linear(), - bias_attr=False) - return out - - -def DeepFM(factor_size, infer=False): - dense_input = paddle.layer.data( - name="dense_input", - type=paddle.data_type.dense_vector(dense_feature_dim)) - sparse_input = paddle.layer.data( - name="sparse_input", - type=paddle.data_type.sparse_binary_vector(sparse_feature_dim)) - sparse_input_ids = [ - paddle.layer.data( - name="C" + str(i), - type=paddle.data_type.integer_value(sparse_feature_dim)) - for i in range(1, 27) - ] - - dense_fm = fm_layer( - dense_input, - factor_size, - fm_param_attr=paddle.attr.Param(name="DenseFeatFactors")) - sparse_fm = fm_layer( - sparse_input, - factor_size, - fm_param_attr=paddle.attr.Param(name="SparseFeatFactors")) - - def embedding_layer(input): - return paddle.layer.embedding( - input=input, - size=factor_size, - param_attr=paddle.attr.Param(name="SparseFeatFactors")) - - sparse_embed_seq = map(embedding_layer, sparse_input_ids) - sparse_embed = paddle.layer.concat(sparse_embed_seq) - - fc1 = paddle.layer.fc(input=[sparse_embed, dense_input], - size=400, - act=paddle.activation.Relu()) - fc2 = paddle.layer.fc(input=fc1, size=400, act=paddle.activation.Relu()) - fc3 = paddle.layer.fc(input=fc2, size=400, act=paddle.activation.Relu()) - - predict = paddle.layer.fc(input=[dense_fm, sparse_fm, fc3], - size=1, - act=paddle.activation.Sigmoid()) - - if not infer: - label = paddle.layer.data( - name="label", type=paddle.data_type.dense_vector(1)) - cost = paddle.layer.multi_binary_label_cross_entropy_cost( - input=predict, label=label) - paddle.evaluator.classification_error( - name="classification_error", input=predict, label=label) - paddle.evaluator.auc(name="auc", input=predict, label=label) - return cost - else: - return predict diff --git a/legacy/deep_fm/preprocess.py b/legacy/deep_fm/preprocess.py deleted file mode 100755 index 36ffea16637c19dee9352d17ed51a67edf582167..0000000000000000000000000000000000000000 --- a/legacy/deep_fm/preprocess.py +++ /dev/null @@ -1,164 +0,0 @@ -""" -Preprocess Criteo dataset. This dataset was used for the Display Advertising -Challenge (https://www.kaggle.com/c/criteo-display-ad-challenge). -""" -import os -import sys -import click -import random -import collections - -# There are 13 integer features and 26 categorical features -continous_features = range(1, 14) -categorial_features = range(14, 40) - -# Clip integer features. The clip point for each integer feature -# is derived from the 95% quantile of the total values in each feature -continous_clip = [20, 600, 100, 50, 64000, 500, 100, 50, 500, 10, 10, 10, 50] - - -class CategoryDictGenerator: - """ - Generate dictionary for each of the categorical features - """ - - def __init__(self, num_feature): - self.dicts = [] - self.num_feature = num_feature - for i in range(0, num_feature): - self.dicts.append(collections.defaultdict(int)) - - def build(self, datafile, categorial_features, cutoff=0): - with open(datafile, 'r') as f: - for line in f: - features = line.rstrip('\n').split('\t') - for i in range(0, self.num_feature): - if features[categorial_features[i]] != '': - self.dicts[i][features[categorial_features[i]]] += 1 - for i in range(0, self.num_feature): - self.dicts[i] = filter(lambda x: x[1] >= cutoff, - self.dicts[i].items()) - self.dicts[i] = sorted(self.dicts[i], key=lambda x: (-x[1], x[0])) - vocabs, _ = list(zip(*self.dicts[i])) - self.dicts[i] = dict(zip(vocabs, range(1, len(vocabs) + 1))) - self.dicts[i][''] = 0 - - def gen(self, idx, key): - if key not in self.dicts[idx]: - res = self.dicts[idx][''] - else: - res = self.dicts[idx][key] - return res - - def dicts_sizes(self): - return map(len, self.dicts) - - -class ContinuousFeatureGenerator: - """ - Normalize the integer features to [0, 1] by min-max normalization - """ - - def __init__(self, num_feature): - self.num_feature = num_feature - self.min = [sys.maxint] * num_feature - self.max = [-sys.maxint] * num_feature - - def build(self, datafile, continous_features): - with open(datafile, 'r') as f: - for line in f: - features = line.rstrip('\n').split('\t') - for i in range(0, self.num_feature): - val = features[continous_features[i]] - if val != '': - val = int(val) - if val > continous_clip[i]: - val = continous_clip[i] - self.min[i] = min(self.min[i], val) - self.max[i] = max(self.max[i], val) - - def gen(self, idx, val): - if val == '': - return 0.0 - val = float(val) - return (val - self.min[idx]) / (self.max[idx] - self.min[idx]) - - -@click.command("preprocess") -@click.option("--datadir", type=str, help="Path to raw criteo dataset") -@click.option("--outdir", type=str, help="Path to save the processed data") -def preprocess(datadir, outdir): - """ - All the 13 integer features are normalzied to continous values and these - continous features are combined into one vecotr with dimension 13. - - Each of the 26 categorical features are one-hot encoded and all the one-hot - vectors are combined into one sparse binary vector. - """ - dists = ContinuousFeatureGenerator(len(continous_features)) - dists.build(os.path.join(datadir, 'train.txt'), continous_features) - - dicts = CategoryDictGenerator(len(categorial_features)) - dicts.build( - os.path.join(datadir, 'train.txt'), categorial_features, cutoff=200) - - dict_sizes = dicts.dicts_sizes() - categorial_feature_offset = [0] - for i in range(1, len(categorial_features)): - offset = categorial_feature_offset[i - 1] + dict_sizes[i - 1] - categorial_feature_offset.append(offset) - - random.seed(0) - - # 90% of the data are used for training, and 10% of the data are used - # for validation. - with open(os.path.join(outdir, 'train.txt'), 'w') as out_train: - with open(os.path.join(outdir, 'valid.txt'), 'w') as out_valid: - with open(os.path.join(datadir, 'train.txt'), 'r') as f: - for line in f: - features = line.rstrip('\n').split('\t') - - continous_vals = [] - for i in range(0, len(continous_features)): - val = dists.gen(i, features[continous_features[i]]) - continous_vals.append("{0:.6f}".format(val).rstrip('0') - .rstrip('.')) - categorial_vals = [] - for i in range(0, len(categorial_features)): - val = dicts.gen(i, features[categorial_features[ - i]]) + categorial_feature_offset[i] - categorial_vals.append(str(val)) - - continous_vals = ','.join(continous_vals) - categorial_vals = ','.join(categorial_vals) - label = features[0] - if random.randint(0, 9999) % 10 != 0: - out_train.write('\t'.join( - [continous_vals, categorial_vals, label]) + '\n') - else: - out_valid.write('\t'.join( - [continous_vals, categorial_vals, label]) + '\n') - - with open(os.path.join(outdir, 'test.txt'), 'w') as out: - with open(os.path.join(datadir, 'test.txt'), 'r') as f: - for line in f: - features = line.rstrip('\n').split('\t') - - continous_vals = [] - for i in range(0, len(continous_features)): - val = dists.gen(i, features[continous_features[i] - 1]) - continous_vals.append("{0:.6f}".format(val).rstrip('0') - .rstrip('.')) - categorial_vals = [] - for i in range(0, len(categorial_features)): - val = dicts.gen(i, features[categorial_features[ - i] - 1]) + categorial_feature_offset[i] - categorial_vals.append(str(val)) - - continous_vals = ','.join(continous_vals) - categorial_vals = ','.join(categorial_vals) - out.write('\t'.join([continous_vals, categorial_vals]) + '\n') - - -if __name__ == "__main__": - preprocess() diff --git a/legacy/deep_fm/reader.py b/legacy/deep_fm/reader.py deleted file mode 100644 index 1098ce423c9071864671be91dea81972e47fbc98..0000000000000000000000000000000000000000 --- a/legacy/deep_fm/reader.py +++ /dev/null @@ -1,58 +0,0 @@ -class Dataset: - def _reader_creator(self, path, is_infer): - def reader(): - with open(path, 'r') as f: - for line in f: - features = line.rstrip('\n').split('\t') - dense_feature = map(float, features[0].split(',')) - sparse_feature = map(int, features[1].split(',')) - if not is_infer: - label = [float(features[2])] - yield [dense_feature, sparse_feature - ] + sparse_feature + [label] - else: - yield [dense_feature, sparse_feature] + sparse_feature - - return reader - - def train(self, path): - return self._reader_creator(path, False) - - def test(self, path): - return self._reader_creator(path, False) - - def infer(self, path): - return self._reader_creator(path, True) - - -feeding = { - 'dense_input': 0, - 'sparse_input': 1, - 'C1': 2, - 'C2': 3, - 'C3': 4, - 'C4': 5, - 'C5': 6, - 'C6': 7, - 'C7': 8, - 'C8': 9, - 'C9': 10, - 'C10': 11, - 'C11': 12, - 'C12': 13, - 'C13': 14, - 'C14': 15, - 'C15': 16, - 'C16': 17, - 'C17': 18, - 'C18': 19, - 'C19': 20, - 'C20': 21, - 'C21': 22, - 'C22': 23, - 'C23': 24, - 'C24': 25, - 'C25': 26, - 'C26': 27, - 'label': 28 -} diff --git a/legacy/deep_fm/train.py b/legacy/deep_fm/train.py deleted file mode 100755 index 92d48696d8845ac13b714b66f7810acdd35fe164..0000000000000000000000000000000000000000 --- a/legacy/deep_fm/train.py +++ /dev/null @@ -1,108 +0,0 @@ -import os -import gzip -import logging -import argparse - -import paddle.v2 as paddle - -from network_conf import DeepFM -import reader - -logging.basicConfig() -logger = logging.getLogger("paddle") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser(description="PaddlePaddle DeepFM example") - parser.add_argument( - '--train_data_path', - type=str, - required=True, - help="The path of training dataset") - parser.add_argument( - '--test_data_path', - type=str, - required=True, - help="The path of testing dataset") - parser.add_argument( - '--batch_size', - type=int, - default=1000, - help="The size of mini-batch (default:1000)") - parser.add_argument( - '--num_passes', - type=int, - default=10, - help="The number of passes to train (default: 10)") - parser.add_argument( - '--factor_size', - type=int, - default=10, - help="The factor size for the factorization machine (default:10)") - parser.add_argument( - '--model_output_dir', - type=str, - default='models', - help='The path for model to store (default: models)') - - return parser.parse_args() - - -def train(): - args = parse_args() - - if not os.path.isdir(args.model_output_dir): - os.mkdir(args.model_output_dir) - - paddle.init(use_gpu=False, trainer_count=1) - - optimizer = paddle.optimizer.Adam(learning_rate=1e-4) - - model = DeepFM(args.factor_size) - - params = paddle.parameters.create(model) - - trainer = paddle.trainer.SGD(cost=model, - parameters=params, - update_equation=optimizer) - - dataset = reader.Dataset() - - def __event_handler__(event): - if isinstance(event, paddle.event.EndIteration): - num_samples = event.batch_id * args.batch_size - if event.batch_id % 100 == 0: - logger.warning("Pass %d, Batch %d, Samples %d, Cost %f, %s" % - (event.pass_id, event.batch_id, num_samples, - event.cost, event.metrics)) - - if event.batch_id % 10000 == 0: - if args.test_data_path: - result = trainer.test( - reader=paddle.batch( - dataset.test(args.test_data_path), - batch_size=args.batch_size), - feeding=reader.feeding) - logger.warning("Test %d-%d, Cost %f, %s" % - (event.pass_id, event.batch_id, result.cost, - result.metrics)) - - path = "{}/model-pass-{}-batch-{}.tar.gz".format( - args.model_output_dir, event.pass_id, event.batch_id) - with gzip.open(path, 'w') as f: - trainer.save_parameter_to_tar(f) - - trainer.train( - reader=paddle.batch( - paddle.reader.shuffle( - dataset.train(args.train_data_path), - buf_size=args.batch_size * 10000), - batch_size=args.batch_size), - feeding=reader.feeding, - event_handler=__event_handler__, - num_passes=args.num_passes) - - -if __name__ == '__main__': - train() diff --git a/legacy/dssm/README.cn.md b/legacy/dssm/README.cn.md deleted file mode 100644 index 140446ad2e071e8bc185d7788dcf33651a370d69..0000000000000000000000000000000000000000 --- a/legacy/dssm/README.cn.md +++ /dev/null @@ -1,294 +0,0 @@ -运行本目录下的程序示例需要使用PaddlePaddle v0.10.0 版本。如果您的PaddlePaddle安装版本低于此版本要求，请按照[安装文档](http://www.paddlepaddle.org/docs/develop/documentation/zh/build_and_install/pip_install_cn.html)中的说明更新PaddlePaddle安装版本。 - ---- - -# 深度结构化语义模型 (Deep Structured Semantic Models, DSSM) -DSSM使用DNN模型在一个连续的语义空间中学习文本低纬的表示向量，并且建模两个句子间的语义相似度。本例演示如何使用PaddlePaddle实现一个通用的DSSM 模型，用于建模两个字符串间的语义相似度，模型实现支持通用的数据格式，用户替换数据便可以在真实场景中使用该模型。 - -## 背景介绍 -DSSM \[[1](##参考文献)\]是微软研究院13年提出来的经典的语义模型，用于学习两个文本之间的语义距离，广义上模型也可以推广和适用如下场景： - -1. CTR预估模型，衡量用户搜索词（Query）与候选网页集合（Documents）之间的相关联程度。 -2. 文本相关性，衡量两个字符串间的语义相关程度。 -3. 自动推荐，衡量User与被推荐的Item之间的关联程度。 - -DSSM 已经发展成了一个框架，可以很自然地建模两个记录之间的距离关系，例如对于文本相关性问题，可以用余弦相似度 (cosin similarity) 来刻画语义距离；而对于搜索引擎的结果排序，可以在DSSM上接上Rank损失训练出一个排序模型。 - -## 模型简介 -在原论文\[[1](#参考文献)\]中，DSSM模型用来衡量用户搜索词 Query 和文档集合 Documents 之间隐含的语义关系，模型结构如下 - -

-

-图 1. DSSM 原始结构 -

- -其贯彻的思想是， **用DNN将高维特征向量转化为低纬空间的连续向量（图中红色框部分）** ，**在上层使用cosine similarity来衡量用户搜索词与候选文档间的语义相关性** 。 - -在最顶层损失函数的设计上，原始模型使用类似Word2Vec中负例采样的方法，一个Query会抽取正例 $D+$ 和4个负例 $D-$ 整体上算条件概率用对数似然函数作为损失，这也就是图 1中类似 $P(D_1|Q)$ 的结构，具体细节请参考原论文。 - -随着后续优化DSSM模型的结构得以简化\[[3](#参考文献)\]，演变为： - -

-

-图 2. DSSM通用结构 -

- -图中的空白方框可以用任何模型替代，例如：全连接FC，卷积CNN，RNN等。该模型结构专门用于衡量两个元素（比如字符串）间的语义距离。在实际任务中，DSSM模型会作为基础的积木，搭配上不同的损失函数来实现具体的功能，比如： - -- 在排序学习中，将 图 2 中结构添加 pairwise rank损失，变成一个排序模型 -- 在CTR预估中，对点击与否做0，1二元分类，添加交叉熵损失变成一个分类模型 -- 在需要对一个子串打分时，可以使用余弦相似度来计算相似度，变成一个回归模型 - -本例提供一个比较通用的解决方案，在模型任务类型上支持： - -- 分类 -- [-1, 1] 值域内的回归 -- Pairwise-Rank - -在生成低纬语义向量的模型结构上，支持以下三种： - -- FC, 多层全连接层 -- CNN，卷积神经网络 -- RNN，递归神经网络 - -## 模型实现 -DSSM模型可以拆成三部分：分别是左边和右边的DNN，以及顶层的损失函数。在复杂任务中，左右两边DNN的结构可以不同。在原始论文中左右网络分别学习Query和Document的语义向量，两者数据的数据不同，建议对应定制DNN的结构。 - -**本例中为了简便和通用，将左右两个DNN的结构设为相同，因此只提供三个选项FC、CNN、RNN**。 - -损失函数的设计也支持三种类型：分类, 回归, 排序；其中，在回归和排序两种损失中，左右两边的匹配程度通过余弦相似度（cosine similairty）来计算；在分类任务中，类别预测的分布通过softmax计算。 - -在其它教程中，对上述很多内容都有过详细的介绍，例如： - -- 如何CNN, FC 做文本信息提取可以参考 [text classification](https://github.com/PaddlePaddle/models/blob/develop/text_classification/README.md#模型详解) -- RNN/GRU 的内容可以参考 [Machine Translation](https://github.com/PaddlePaddle/book/blob/develop/08.machine_translation/README.md#gated-recurrent-unit-gru) -- Pairwise Rank即排序学习可参考 [learn to rank](https://github.com/PaddlePaddle/models/blob/develop/ltr/README.md) - -相关原理在此不再赘述，本文接下来的篇幅主要集中介绍使用PaddlePaddle实现这些结构上。 - -如图3，回归和分类模型的结构相似: - -

-

-图 3. DSSM for REGRESSION or CLASSIFICATION -

- -最重要的组成部分包括词向量，图中`(1)`,`(2)`两个低纬向量的学习器（可以用RNN/CNN/FC中的任意一种实现），最上层对应的损失函数。 - -Pairwise Rank的结构会复杂一些，图 4. 中的结构会出现两次，增加了对应的损失函数，模型总体思想是： -- 给定同一个source(源)为左右两个target(目标)分别打分——`(a),(b)`，学习目标是(a),(b)之间的大小关系 -- `(a)`和`(b)`类似图3中结构，用于给source和target的pair打分 -- `(1)`和`(2)`的结构其实是共用的，都表示同一个source，图中为了表达效果展开成两个 - -

-

-图 4. DSSM for Pairwise Rank -

- -下面是各个部分的具体实现，相关代码均包含在 `./network_conf.py` 中。 - - -### 创建文本的词向量表 - -```python -def create_embedding(self, input, prefix=''): - """ - Create word embedding. The `prefix` is added in front of the name of - embedding"s learnable parameter. - """ - logger.info("Create embedding table [%s] whose dimention is %d" % - (prefix, self.dnn_dims[0])) - emb = paddle.layer.embedding( - input=input, - size=self.dnn_dims[0], - param_attr=ParamAttr(name='%s_emb.w' % prefix)) - return emb -``` - -由于输入给词向量表(embedding table)的是一个句子对应的词的ID的列表 ，因此词向量表输出的是词向量的序列。 - -### CNN 结构实现 - -```python -def create_cnn(self, emb, prefix=''): - - """ - A multi-layer CNN. - :param emb: The word embedding. - :type emb: paddle.layer - :param prefix: The prefix will be added to of layers' names. - :type prefix: str - """ - - def create_conv(context_len, hidden_size, prefix): - key = "%s_%d_%d" % (prefix, context_len, hidden_size) - conv = paddle.networks.sequence_conv_pool( - input=emb, - context_len=context_len, - hidden_size=hidden_size, - # set parameter attr for parameter sharing - context_proj_param_attr=ParamAttr(name=key + "contex_proj.w"), - fc_param_attr=ParamAttr(name=key + "_fc.w"), - fc_bias_attr=ParamAttr(name=key + "_fc.b"), - pool_bias_attr=ParamAttr(name=key + "_pool.b")) - return conv - - conv_3 = create_conv(3, self.dnn_dims[1], "cnn") - conv_4 = create_conv(4, self.dnn_dims[1], "cnn") - return paddle.layer.concat(input=[conv_3, conv_4]) -``` - -CNN 接受词向量序列，通过卷积和池化操作捕捉到原始句子的关键信息，最终输出一个语义向量（可以认为是句子向量）。 - -本例的实现中，分别使用了窗口长度为3和4的CNN学到的句子向量按元素求和得到最终的句子向量。 - -### RNN 结构实现 - -RNN很适合学习变长序列的信息，使用RNN来学习句子的信息几乎是自然语言处理任务的标配。 - -```python -def create_rnn(self, emb, prefix=''): - """ - A GRU sentence vector learner. - """ - gru = paddle.networks.simple_gru( - input=emb, - size=self.dnn_dims[1], - mixed_param_attr=ParamAttr(name='%s_gru_mixed.w' % prefix), - mixed_bias_param_attr=ParamAttr(name="%s_gru_mixed.b" % prefix), - gru_param_attr=ParamAttr(name='%s_gru.w' % prefix), - gru_bias_attr=ParamAttr(name="%s_gru.b" % prefix)) - sent_vec = paddle.layer.last_seq(gru) - return sent_vec -``` - -### 多层全连接网络FC - -```python -def create_fc(self, emb, prefix=''): - - """ - A multi-layer fully connected neural networks. - :param emb: The output of the embedding layer - :type emb: paddle.layer - :param prefix: A prefix will be added to the layers' names. - :type prefix: str - """ - - _input_layer = paddle.layer.pooling( - input=emb, pooling_type=paddle.pooling.Max()) - fc = paddle.layer.fc( - input=_input_layer, - size=self.dnn_dims[1], - param_attr=ParamAttr(name='%s_fc.w' % prefix), - bias_attr=ParamAttr(name="%s_fc.b" % prefix)) - return fc -``` - -在构建全连接网络时首先使用`paddle.layer.pooling` 对词向量序列进行最大池化操作，将边长序列转化为一个固定维度向量，作为整个句子的语义表达，使用最大池化能够降低句子长度对句向量表达的影响。 - -### 多层DNN -在 CNN/DNN/FC提取出 semantic vector后，在上层可继续接多层FC来实现深层DNN结构。 - -```python -def create_dnn(self, sent_vec, prefix): - if len(self.dnn_dims) > 1: - _input_layer = sent_vec - for id, dim in enumerate(self.dnn_dims[1:]): - name = "%s_fc_%d_%d" % (prefix, id, dim) - fc = paddle.layer.fc( - input=_input_layer, - size=dim, - act=paddle.activation.Tanh(), - param_attr=ParamAttr(name='%s.w' % name), - bias_attr=ParamAttr(name='%s.b' % name), - ) - _input_layer = fc - return _input_layer -``` - -### 分类及回归 -分类和回归的结构比较相似，具体实现请参考[network_conf.py]( https://github.com/PaddlePaddle/models/blob/develop/dssm/network_conf.py)中的 -`_build_classification_or_regression_model` 函数。 - -### Pairwise Rank -Pairwise Rank复用上面的DNN结构，同一个source对两个target求相似度打分，如果左边的target打分高，预测为1，否则预测为 0。实现请参考 [network_conf.py]( https://github.com/PaddlePaddle/models/blob/develop/dssm/network_conf.py) 中的`_build_rank_model` 函数。 - -## 数据格式 -在 `./data` 中有简单的示例数据 - -### 回归的数据格式 -``` -# 3 fields each line: -# - source word list -# - target word list -# - target - \t \t -``` - -比如： - -``` -苹果 六 袋 苹果 6s 0.1 -新手 汽车 驾驶 驾校 培训 0.9 -``` -### 分类的数据格式 -``` -# 3 fields each line: -# - source word list -# - target word list -# - target - \t \t