diff --git a/models/multitask/esmm/__init__.py b/models/multitask/esmm/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..abf198b97e6e818e1fbe59006f98492640bcee54 --- /dev/null +++ b/models/multitask/esmm/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/multitask/esmm/config.yaml b/models/multitask/esmm/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cc16a5d14ee077d962db1de39d7f4f405a43649 --- /dev/null +++ b/models/multitask/esmm/config.yaml @@ -0,0 +1,44 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +train: + trainer: + # for cluster training + strategy: "async" + + epochs: 3 + workspace: "fleetrec.models.multitask.esmm" + + reader: + batch_size: 2 + class: "{workspace}/esmm_reader.py" + train_data_path: "{workspace}/data/train" + + model: + models: "{workspace}/model.py" + hyper_parameters: + vocab_size: 10000 + embed_size: 128 + learning_rate: 0.001 + optimizer: adam + + save: + increment: + dirname: "increment" + epoch_interval: 2 + save_last: True + inference: + dirname: "inference" + epoch_interval: 4 + save_last: True diff --git a/models/multitask/esmm/data/test/small_test.txt b/models/multitask/esmm/data/test/small_test.txt new file mode 100644 index 0000000000000000000000000000000000000000..b4bf7189643a041b769fc88c56f6b1ec5b5229db --- /dev/null +++ b/models/multitask/esmm/data/test/small_test.txt @@ -0,0 +1,100 @@ +0 16 +475 473 155 +491 21 +96 185 96 +29 14 13 +5 481 11 21 470 +70 5 70 11 +167 42 167 217 +72 15 73 161 172 +82 82 +97 297 97 +193 182 186 183 184 177 214 +152 152 +163 298 7 +39 73 71 +490 23 23 496 488 74 23 74 486 23 23 74 +17 17 +170 170 483 444 443 234 +25 472 +5 5 11 70 69 +149 149 455 +356 68 477 468 17 479 66 +159 172 6 71 6 6 158 13 494 169 +155 44 438 144 500 +156 9 9 +146 146 +173 10 10 461 +7 6 6 +269 48 268 +50 100 +323 174 18 +69 69 22 98 +38 171 +22 29 489 10 +0 0 +11 5 +29 13 14 232 231 451 289 452 229 +260 11 156 +166 160 166 39 +223 134 134 420 +66 401 68 132 17 84 287 5 +39 304 +65 84 132 +400 211 +145 144 +16 28 254 48 50 100 42 154 262 133 17 +0 0 +28 28 +11 476 464 +61 61 86 86 +38 38 +463 478 +437 265 +22 39 485 171 98 +434 51 344 +16 16 +67 67 67 448 +22 12 161 +15 377 147 147 374 +119 317 0 +38 484 +403 499 +432 442 +28 0 16 50 465 42 +163 487 7 162 +99 99 325 423 83 83 +154 133 +5 37 492 235 160 279 +10 10 457 493 10 460 +441 4 4 4 4 4 4 4 +153 153 +159 164 164 +328 37 +65 65 404 347 431 459 +80 80 44 44 +61 446 +162 495 7 453 +157 21 204 68 37 66 469 145 +37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 +430 445 433 +449 14 +9 9 9 9 +440 238 226 +148 148 +266 267 181 +48 498 +263 255 256 +458 158 7 +72 168 12 165 71 73 173 49 +0 0 +7 7 6 +14 29 13 6 15 14 15 13 +480 439 21 +450 21 151 +12 12 49 14 13 165 12 169 72 15 15 +91 91 +22 12 49 168 +497 101 30 411 30 482 30 53 30 101 176 415 53 447 +462 150 150 +471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 diff --git a/models/multitask/esmm/data/train/small.csv b/models/multitask/esmm/data/train/small.csv new file mode 100644 index 0000000000000000000000000000000000000000..f5febad64a6a0e73313ea59c8c5f7822aa72fb02 --- /dev/null +++ b/models/multitask/esmm/data/train/small.csv @@ -0,0 +1,10 @@ +1,0,0,210:1,210:2,210:3,216:4,301:5,205:6,206:7,207:8,508:9,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 +2,0,0,210:877,210:878,210:879,210:880,210:881,216:882,301:5,205:883,206:884,207:885,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 +3,1,0,210:886,210:887,509:888,508:889,702:890,853:891,207:892,206:893,205:894,301:5,216:895,210:896,210:897,210:898,210:899,210:900,210:901,210:902,210:903,210:904,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 +4,0,0,301:5,210:905,210:906,210:907,210:908,508:909,853:910,853:911,853:912,207:913,206:914,205:915,216:916,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 +5,0,0,205:917,301:5,216:918,210:919,210:920,210:921,508:922,207:923,206:924,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 +6,0,0,210:897,210:925,210:926,210:927,210:928,210:929,210:930,210:931,210:901,210:932,210:933,210:934,210:935,210:936,216:937,301:5,205:938,206:893,207:939,853:891,508:889,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 +7,0,0,210:887,301:5,205:940,206:893,207:941,853:891,508:889,210:901,210:925,210:897,210:931,210:929,216:937,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 +8,0,0,216:942,210:943,210:944,509:945,508:946,702:947,207:948,206:949,205:950,301:5,210:951,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 +9,0,0,206:952,205:953,301:5,216:954,210:955,210:956,210:957,509:958,207:959,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 +10,0,0,216:942,210:960,210:961,210:962,508:963,702:947,207:964,206:965,205:966,301:5,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876 diff --git a/models/multitask/esmm/esmm_reader.py b/models/multitask/esmm/esmm_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..a18702d39ae922ffd881c11966df0dfe8b713f51 --- /dev/null +++ b/models/multitask/esmm/esmm_reader.py @@ -0,0 +1,66 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function + +from fleetrec.core.reader import Reader +from fleetrec.core.utils import envs +from collections import defaultdict +import numpy as np + + +class TrainReader(Reader): + def init(self): + all_field_id = ['101', '109_14', '110_14', '127_14', '150_14', '121', '122', '124', '125', '126', '127', '128', '129', + '205', '206', '207', '210', '216', '508', '509', '702', '853', '301'] + self.all_field_id_dict = defaultdict(int) + for i,field_id in enumerate(all_field_id): + self.all_field_id_dict[field_id] = [False,i] + + def generate_sample(self, line): + """ + Read the data line by line and process it as a dictionary + """ + + def reader(): + """ + This function needs to be implemented by the user, based on data format + """ + features = line.strip().split(',') + #ctr = list(map(int, features[1])) + #cvr = list(map(int, features[2])) + ctr = int(features[1]) + cvr = int(features[2]) + + padding = 0 + output = [(field_id,[]) for field_id in self.all_field_id_dict] + + for elem in features[4:]: + field_id,feat_id = elem.strip().split(':') + if field_id not in self.all_field_id_dict: + continue + self.all_field_id_dict[field_id][0] = True + index = self.all_field_id_dict[field_id][1] + #feat_id = list(map(int, feat_id)) + output[index][1].append(int(feat_id)) + + for field_id in self.all_field_id_dict: + visited,index = self.all_field_id_dict[field_id] + if visited: + self.all_field_id_dict[field_id][0] = False + else: + output[index][1].append(padding) + output.append(('ctr', [ctr])) + output.append(('cvr', [cvr])) + yield output + return reader diff --git a/models/multitask/esmm/model.py b/models/multitask/esmm/model.py new file mode 100644 index 0000000000000000000000000000000000000000..0063cdfefd7abebb315750d13fa723207fb5eb08 --- /dev/null +++ b/models/multitask/esmm/model.py @@ -0,0 +1,114 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import paddle.fluid as fluid + +from fleetrec.core.utils import envs +from fleetrec.core.model import Model as ModelBase +import numpy as np + + +class Model(ModelBase): + def __init__(self, config): + ModelBase.__init__(self, config) + + def fc(self,tag, data, out_dim, active='prelu'): + + init_stddev = 1.0 + scales = 1.0 / np.sqrt(data.shape[1]) + + p_attr = fluid.param_attr.ParamAttr(name='%s_weight' % tag, + initializer=fluid.initializer.NormalInitializer(loc=0.0, scale=init_stddev * scales)) + + b_attr = fluid.ParamAttr(name='%s_bias' % tag, initializer=fluid.initializer.Constant(0.1)) + + out = fluid.layers.fc(input=data, + size=out_dim, + act=active, + param_attr=p_attr, + bias_attr =b_attr, + name=tag) + return out + + def input_data(self): + sparse_input_ids = [ + fluid.data(name="field_" + str(i), shape=[-1, 1], dtype="int64", lod_level=1) for i in range(0,23) + ] + label_ctr = fluid.data(name="ctr", shape=[-1, 1], dtype="int64") + label_cvr = fluid.data(name="cvr", shape=[-1, 1], dtype="int64") + inputs = sparse_input_ids + [label_ctr] + [label_cvr] + self._data_var.extend(inputs) + + return inputs + + def net(self, inputs): + + vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace) + embed_size = envs.get_global_env("hyper_parameters.embed_size", None, self._namespace) + emb = [] + for data in inputs[0:-2]: + feat_emb = fluid.embedding(input=data, + size=[vocab_size, embed_size], + param_attr=fluid.ParamAttr(name='dis_emb', + learning_rate=5, + initializer=fluid.initializer.Xavier(fan_in=embed_size,fan_out=embed_size) + ), + is_sparse=True) + field_emb = fluid.layers.sequence_pool(input=feat_emb,pool_type='sum') + emb.append(field_emb) + concat_emb = fluid.layers.concat(emb, axis=1) + + # ctr + active = 'relu' + ctr_fc1 = self.fc('ctr_fc1', concat_emb, 200, active) + ctr_fc2 = self.fc('ctr_fc2', ctr_fc1, 80, active) + ctr_out = self.fc('ctr_out', ctr_fc2, 2, 'softmax') + + # cvr + cvr_fc1 = self.fc('cvr_fc1', concat_emb, 200, active) + cvr_fc2 = self.fc('cvr_fc2', cvr_fc1, 80, active) + cvr_out = self.fc('cvr_out', cvr_fc2, 2,'softmax') + + ctr_clk = inputs[-2] + ctcvr_buy = inputs[-1] + + ctr_prop_one = fluid.layers.slice(ctr_out, axes=[1], starts=[1], ends=[2]) + cvr_prop_one = fluid.layers.slice(cvr_out, axes=[1], starts=[1], ends=[2]) + + ctcvr_prop_one = fluid.layers.elementwise_mul(ctr_prop_one, cvr_prop_one) + ctcvr_prop = fluid.layers.concat(input=[1-ctcvr_prop_one,ctcvr_prop_one], axis = 1) + + loss_ctr = fluid.layers.cross_entropy(input=ctr_out, label=ctr_clk) + loss_ctcvr = fluid.layers.cross_entropy(input=ctcvr_prop, label=ctcvr_buy) + cost = loss_ctr + loss_ctcvr + avg_cost = fluid.layers.mean(cost) + + auc_ctr, batch_auc_ctr, auc_states_ctr = fluid.layers.auc(input=ctr_out, label=ctr_clk) + auc_ctcvr, batch_auc_ctcvr, auc_states_ctcvr = fluid.layers.auc(input=ctcvr_prop, label=ctcvr_buy) + + self._cost = avg_cost + self._metrics["AUC_ctr"] = auc_ctr + self._metrics["BATCH_AUC_ctr"] = batch_auc_ctr + self._metrics["AUC_ctcvr"] = auc_ctcvr + self._metrics["BATCH_AUC_ctcvr"] = batch_auc_ctcvr + + + def train_net(self): + input_data = self.input_data() + self.net(input_data) + + + def infer_net(self): + pass diff --git a/models/multitask/mmoe/__init__.py b/models/multitask/mmoe/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..abf198b97e6e818e1fbe59006f98492640bcee54 --- /dev/null +++ b/models/multitask/mmoe/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/multitask/mmoe/census_reader.py b/models/multitask/mmoe/census_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..323c15a40a29070285b89351612b1efdf162cd2c --- /dev/null +++ b/models/multitask/mmoe/census_reader.py @@ -0,0 +1,52 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function + +from fleetrec.core.reader import Reader +from fleetrec.core.utils import envs +import numpy as np + + +class TrainReader(Reader): + def init(self): + pass + + def generate_sample(self, line): + """ + Read the data line by line and process it as a dictionary + """ + + def reader(): + """ + This function needs to be implemented by the user, based on data format + """ + l = line.strip().split(',') + l = list(map(float, l)) + label_income = [] + label_marital = [] + data = l[2:] + if int(l[1]) == 0: + label_income = [1, 0] + elif int(l[1]) == 1: + label_income = [0, 1] + if int(l[0]) == 0: + label_marital = [1, 0] + elif int(l[0]) == 1: + label_marital = [0, 1] + #label_income = np.array(label_income) + #label_marital = np.array(label_marital) + feature_name = ["input", "label_income", "label_marital"] + yield zip(feature_name, [data] + [label_income] + [label_marital]) + + return reader diff --git a/models/multitask/mmoe/config.yaml b/models/multitask/mmoe/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca6552d8045761792b362fa3345e886cde396abb --- /dev/null +++ b/models/multitask/mmoe/config.yaml @@ -0,0 +1,47 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +train: + trainer: + # for cluster training + strategy: "async" + + epochs: 3 + workspace: "fleetrec.models.multitask.mmoe" + + reader: + batch_size: 2 + class: "{workspace}/census_reader.py" + train_data_path: "{workspace}/data/train" + + model: + models: "{workspace}/model.py" + hyper_parameters: + feature_size: 499 + expert_num: 8 + gate_num: 2 + expert_size: 16 + tower_size: 8 + learning_rate: 0.001 + optimizer: adam + + save: + increment: + dirname: "increment" + epoch_interval: 2 + save_last: True + inference: + dirname: "inference" + epoch_interval: 4 + save_last: True diff --git a/models/multitask/mmoe/data/test/small_test.txt b/models/multitask/mmoe/data/test/small_test.txt new file mode 100644 index 0000000000000000000000000000000000000000..b4bf7189643a041b769fc88c56f6b1ec5b5229db --- /dev/null +++ b/models/multitask/mmoe/data/test/small_test.txt @@ -0,0 +1,100 @@ +0 16 +475 473 155 +491 21 +96 185 96 +29 14 13 +5 481 11 21 470 +70 5 70 11 +167 42 167 217 +72 15 73 161 172 +82 82 +97 297 97 +193 182 186 183 184 177 214 +152 152 +163 298 7 +39 73 71 +490 23 23 496 488 74 23 74 486 23 23 74 +17 17 +170 170 483 444 443 234 +25 472 +5 5 11 70 69 +149 149 455 +356 68 477 468 17 479 66 +159 172 6 71 6 6 158 13 494 169 +155 44 438 144 500 +156 9 9 +146 146 +173 10 10 461 +7 6 6 +269 48 268 +50 100 +323 174 18 +69 69 22 98 +38 171 +22 29 489 10 +0 0 +11 5 +29 13 14 232 231 451 289 452 229 +260 11 156 +166 160 166 39 +223 134 134 420 +66 401 68 132 17 84 287 5 +39 304 +65 84 132 +400 211 +145 144 +16 28 254 48 50 100 42 154 262 133 17 +0 0 +28 28 +11 476 464 +61 61 86 86 +38 38 +463 478 +437 265 +22 39 485 171 98 +434 51 344 +16 16 +67 67 67 448 +22 12 161 +15 377 147 147 374 +119 317 0 +38 484 +403 499 +432 442 +28 0 16 50 465 42 +163 487 7 162 +99 99 325 423 83 83 +154 133 +5 37 492 235 160 279 +10 10 457 493 10 460 +441 4 4 4 4 4 4 4 +153 153 +159 164 164 +328 37 +65 65 404 347 431 459 +80 80 44 44 +61 446 +162 495 7 453 +157 21 204 68 37 66 469 145 +37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 +430 445 433 +449 14 +9 9 9 9 +440 238 226 +148 148 +266 267 181 +48 498 +263 255 256 +458 158 7 +72 168 12 165 71 73 173 49 +0 0 +7 7 6 +14 29 13 6 15 14 15 13 +480 439 21 +450 21 151 +12 12 49 14 13 165 12 169 72 15 15 +91 91 +22 12 49 168 +497 101 30 411 30 482 30 53 30 101 176 415 53 447 +462 150 150 +471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 diff --git a/models/multitask/mmoe/data/train/train_data b/models/multitask/mmoe/data/train/train_data new file mode 100644 index 0000000000000000000000000000000000000000..992314e443942c1b3e08a7db88bf2c1d7354c451 --- /dev/null +++ b/models/multitask/mmoe/data/train/train_data @@ -0,0 +1,10 @@ +0,0,73,0,0,0,0,1700.09,0,0,2,0,95,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,58,0,0,0,0,1053.55,1,0,2,52,94,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +1,0,18,0,0,0,0,991.95,0,0,2,0,95,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0 +1,0,9,0,0,0,0,1758.14,0,0,0,0,94,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +1,0,10,0,0,0,0,1069.16,0,0,0,0,94,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,48,1200,0,0,0,162.61,1,2,2,52,95,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,42,0,5178,0,0,1535.86,6,0,2,52,94,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +1,0,28,0,0,0,0,898.83,4,0,2,30,95,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,47,876,0,0,0,1661.53,5,0,2,52,95,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,34,0,0,0,0,1146.79,6,0,2,52,94,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 diff --git a/models/multitask/mmoe/model.py b/models/multitask/mmoe/model.py new file mode 100644 index 0000000000000000000000000000000000000000..0b757d0270e7f5159bf4154e66b8e321fb9d9885 --- /dev/null +++ b/models/multitask/mmoe/model.py @@ -0,0 +1,104 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import paddle.fluid as fluid + +from fleetrec.core.utils import envs +from fleetrec.core.model import Model as ModelBase + + +class Model(ModelBase): + def __init__(self, config): + ModelBase.__init__(self, config) + + def MMOE(self): + + feature_size = envs.get_global_env("hyper_parameters.feature_size", None, self._namespace) + expert_num = envs.get_global_env("hyper_parameters.expert_num", None, self._namespace) + gate_num = envs.get_global_env("hyper_parameters.gate_num", None, self._namespace) + expert_size = envs.get_global_env("hyper_parameters.expert_size", None, self._namespace) + tower_size = envs.get_global_env("hyper_parameters.tower_size", None, self._namespace) + + input_data = fluid.data(name="input", shape=[-1, feature_size], dtype="float32") + label_income = fluid.data(name="label_income", shape=[-1, 2], dtype="float32", lod_level=0) + label_marital = fluid.data(name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0) + + self._data_var.extend([input_data, label_income, label_marital]) + # f_{i}(x) = activation(W_{i} * x + b), where activation is ReLU according to the paper + expert_outputs = [] + for i in range(0, expert_num): + expert_output = fluid.layers.fc(input=input_data, + size=expert_size, + act='relu', + bias_attr=fluid.ParamAttr(learning_rate=1.0), + name='expert_' + str(i)) + expert_outputs.append(expert_output) + expert_concat = fluid.layers.concat(expert_outputs, axis=1) + expert_concat = fluid.layers.reshape(expert_concat,[-1, expert_num, expert_size]) + + + # g^{k}(x) = activation(W_{gk} * x + b), where activation is softmax according to the paper + output_layers = [] + for i in range(0, gate_num): + cur_gate = fluid.layers.fc(input=input_data, + size=expert_num, + act='softmax', + bias_attr=fluid.ParamAttr(learning_rate=1.0), + name='gate_' + str(i)) + # f^{k}(x) = sum_{i=1}^{n}(g^{k}(x)_{i} * f_{i}(x)) + cur_gate_expert = fluid.layers.elementwise_mul(expert_concat, cur_gate, axis=0) + cur_gate_expert = fluid.layers.reduce_sum(cur_gate_expert, dim=1) + # Build tower layer + cur_tower = fluid.layers.fc(input=cur_gate_expert, + size=tower_size, + act='relu', + name='task_layer_' + str(i)) + out = fluid.layers.fc(input=cur_tower, + size=2, + act='softmax', + name='out_' + str(i)) + + output_layers.append(out) + + pred_income = fluid.layers.clip(output_layers[0], min=1e-15, max=1.0 - 1e-15) + pred_marital = fluid.layers.clip(output_layers[1], min=1e-15, max=1.0 - 1e-15) + + cost_income = fluid.layers.cross_entropy(input=pred_income, label=label_income,soft_label = True) + cost_marital = fluid.layers.cross_entropy(input=pred_marital, label=label_marital,soft_label = True) + + label_income_1 = fluid.layers.slice(label_income, axes=[1], starts=[1], ends=[2]) + label_marital_1 = fluid.layers.slice(label_marital, axes=[1], starts=[1], ends=[2]) + + auc_income, batch_auc_1, auc_states_1 = fluid.layers.auc(input=pred_income, label=fluid.layers.cast(x=label_income_1, dtype='int64')) + auc_marital, batch_auc_2, auc_states_2 = fluid.layers.auc(input=pred_marital, label=fluid.layers.cast(x=label_marital_1, dtype='int64')) + + avg_cost_income = fluid.layers.mean(x=cost_income) + avg_cost_marital = fluid.layers.mean(x=cost_marital) + + cost = avg_cost_income + avg_cost_marital + + self._cost = cost + self._metrics["AUC_income"] = auc_income + self._metrics["BATCH_AUC_income"] = batch_auc_1 + self._metrics["AUC_marital"] = auc_marital + self._metrics["BATCH_AUC_marital"] = batch_auc_2 + + + def train_net(self): + self.MMOE() + + + def infer_net(self): + pass diff --git a/models/multitask/share-bottom/__init__.py b/models/multitask/share-bottom/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..abf198b97e6e818e1fbe59006f98492640bcee54 --- /dev/null +++ b/models/multitask/share-bottom/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/multitask/share-bottom/census_reader.py b/models/multitask/share-bottom/census_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..323c15a40a29070285b89351612b1efdf162cd2c --- /dev/null +++ b/models/multitask/share-bottom/census_reader.py @@ -0,0 +1,52 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function + +from fleetrec.core.reader import Reader +from fleetrec.core.utils import envs +import numpy as np + + +class TrainReader(Reader): + def init(self): + pass + + def generate_sample(self, line): + """ + Read the data line by line and process it as a dictionary + """ + + def reader(): + """ + This function needs to be implemented by the user, based on data format + """ + l = line.strip().split(',') + l = list(map(float, l)) + label_income = [] + label_marital = [] + data = l[2:] + if int(l[1]) == 0: + label_income = [1, 0] + elif int(l[1]) == 1: + label_income = [0, 1] + if int(l[0]) == 0: + label_marital = [1, 0] + elif int(l[0]) == 1: + label_marital = [0, 1] + #label_income = np.array(label_income) + #label_marital = np.array(label_marital) + feature_name = ["input", "label_income", "label_marital"] + yield zip(feature_name, [data] + [label_income] + [label_marital]) + + return reader diff --git a/models/multitask/share-bottom/config.yaml b/models/multitask/share-bottom/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..432ebdf331f098580dfad2019c05af3e4e47553d --- /dev/null +++ b/models/multitask/share-bottom/config.yaml @@ -0,0 +1,46 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +train: + trainer: + # for cluster training + strategy: "async" + + epochs: 3 + workspace: "fleetrec.models.multitask.share-bottom" + + reader: + batch_size: 2 + class: "{workspace}/census_reader.py" + train_data_path: "{workspace}/data/train" + + model: + models: "{workspace}/model.py" + hyper_parameters: + feature_size: 499 + bottom_size: 117 + tower_nums: 2 + tower_size: 8 + learning_rate: 0.001 + optimizer: adam + + save: + increment: + dirname: "increment" + epoch_interval: 2 + save_last: True + inference: + dirname: "inference" + epoch_interval: 4 + save_last: True diff --git a/models/multitask/share-bottom/data/test/small_test.txt b/models/multitask/share-bottom/data/test/small_test.txt new file mode 100644 index 0000000000000000000000000000000000000000..b4bf7189643a041b769fc88c56f6b1ec5b5229db --- /dev/null +++ b/models/multitask/share-bottom/data/test/small_test.txt @@ -0,0 +1,100 @@ +0 16 +475 473 155 +491 21 +96 185 96 +29 14 13 +5 481 11 21 470 +70 5 70 11 +167 42 167 217 +72 15 73 161 172 +82 82 +97 297 97 +193 182 186 183 184 177 214 +152 152 +163 298 7 +39 73 71 +490 23 23 496 488 74 23 74 486 23 23 74 +17 17 +170 170 483 444 443 234 +25 472 +5 5 11 70 69 +149 149 455 +356 68 477 468 17 479 66 +159 172 6 71 6 6 158 13 494 169 +155 44 438 144 500 +156 9 9 +146 146 +173 10 10 461 +7 6 6 +269 48 268 +50 100 +323 174 18 +69 69 22 98 +38 171 +22 29 489 10 +0 0 +11 5 +29 13 14 232 231 451 289 452 229 +260 11 156 +166 160 166 39 +223 134 134 420 +66 401 68 132 17 84 287 5 +39 304 +65 84 132 +400 211 +145 144 +16 28 254 48 50 100 42 154 262 133 17 +0 0 +28 28 +11 476 464 +61 61 86 86 +38 38 +463 478 +437 265 +22 39 485 171 98 +434 51 344 +16 16 +67 67 67 448 +22 12 161 +15 377 147 147 374 +119 317 0 +38 484 +403 499 +432 442 +28 0 16 50 465 42 +163 487 7 162 +99 99 325 423 83 83 +154 133 +5 37 492 235 160 279 +10 10 457 493 10 460 +441 4 4 4 4 4 4 4 +153 153 +159 164 164 +328 37 +65 65 404 347 431 459 +80 80 44 44 +61 446 +162 495 7 453 +157 21 204 68 37 66 469 145 +37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 +430 445 433 +449 14 +9 9 9 9 +440 238 226 +148 148 +266 267 181 +48 498 +263 255 256 +458 158 7 +72 168 12 165 71 73 173 49 +0 0 +7 7 6 +14 29 13 6 15 14 15 13 +480 439 21 +450 21 151 +12 12 49 14 13 165 12 169 72 15 15 +91 91 +22 12 49 168 +497 101 30 411 30 482 30 53 30 101 176 415 53 447 +462 150 150 +471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 diff --git a/models/multitask/share-bottom/data/train/train_data b/models/multitask/share-bottom/data/train/train_data new file mode 100644 index 0000000000000000000000000000000000000000..992314e443942c1b3e08a7db88bf2c1d7354c451 --- /dev/null +++ b/models/multitask/share-bottom/data/train/train_data @@ -0,0 +1,10 @@ +0,0,73,0,0,0,0,1700.09,0,0,2,0,95,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,58,0,0,0,0,1053.55,1,0,2,52,94,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +1,0,18,0,0,0,0,991.95,0,0,2,0,95,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0 +1,0,9,0,0,0,0,1758.14,0,0,0,0,94,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +1,0,10,0,0,0,0,1069.16,0,0,0,0,94,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,48,1200,0,0,0,162.61,1,2,2,52,95,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,42,0,5178,0,0,1535.86,6,0,2,52,94,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +1,0,28,0,0,0,0,898.83,4,0,2,30,95,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,47,876,0,0,0,1661.53,5,0,2,52,95,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 +0,0,34,0,0,0,0,1146.79,6,0,2,52,94,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0 diff --git a/models/multitask/share-bottom/model.py b/models/multitask/share-bottom/model.py new file mode 100644 index 0000000000000000000000000000000000000000..a364f1b82b9f4a6849e67cd1ff34fa8d179b475c --- /dev/null +++ b/models/multitask/share-bottom/model.py @@ -0,0 +1,89 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import paddle.fluid as fluid + +from fleetrec.core.utils import envs +from fleetrec.core.model import Model as ModelBase + + +class Model(ModelBase): + def __init__(self, config): + ModelBase.__init__(self, config) + + def train(self): + + feature_size = envs.get_global_env("hyper_parameters.feature_size", None, self._namespace) + bottom_size = envs.get_global_env("hyper_parameters.bottom_size", None, self._namespace) + tower_size = envs.get_global_env("hyper_parameters.tower_size", None, self._namespace) + tower_nums = envs.get_global_env("hyper_parameters.tower_nums", None, self._namespace) + + input_data = fluid.data(name="input", shape=[-1, feature_size], dtype="float32") + label_income = fluid.data(name="label_income", shape=[-1, 2], dtype="float32", lod_level=0) + label_marital = fluid.data(name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0) + + self._data_var.extend([input_data, label_income, label_marital]) + + bottom_output = fluid.layers.fc(input=input_data, + size=bottom_size, + act='relu', + bias_attr=fluid.ParamAttr(learning_rate=1.0), + name='bottom_output') + + + # Build tower layer from bottom layer + output_layers = [] + for index in range(tower_nums): + tower_layer = fluid.layers.fc(input=bottom_output, + size=tower_size, + act='relu', + name='task_layer_' + str(index)) + output_layer = fluid.layers.fc(input=tower_layer, + size=2, + act='softmax', + name='output_layer_' + str(index)) + output_layers.append(output_layer) + + + pred_income = fluid.layers.clip(output_layers[0], min=1e-15, max=1.0 - 1e-15) + pred_marital = fluid.layers.clip(output_layers[1], min=1e-15, max=1.0 - 1e-15) + + cost_income = fluid.layers.cross_entropy(input=pred_income, label=label_income,soft_label = True) + cost_marital = fluid.layers.cross_entropy(input=pred_marital, label=label_marital,soft_label = True) + + + label_income_1 = fluid.layers.slice(label_income, axes=[1], starts=[1], ends=[2]) + label_marital_1 = fluid.layers.slice(label_marital, axes=[1], starts=[1], ends=[2]) + + auc_income, batch_auc_1, auc_states_1 = fluid.layers.auc(input=pred_income, label=fluid.layers.cast(x=label_income_1, dtype='int64')) + auc_marital, batch_auc_2, auc_states_2 = fluid.layers.auc(input=pred_marital, label=fluid.layers.cast(x=label_marital_1, dtype='int64')) + + cost = fluid.layers.elementwise_add(cost_income, cost_marital, axis=1) + + avg_cost = fluid.layers.mean(x=cost) + + self._cost = avg_cost + self._metrics["AUC_income"] = auc_income + self._metrics["BATCH_AUC_income"] = batch_auc_1 + self._metrics["AUC_marital"] = auc_marital + self._metrics["BATCH_AUC_marital"] = batch_auc_2 + + + def train_net(self): + self.train() + + + def infer_net(self): + pass diff --git a/models/recall/gru4rec/__init__.py b/models/recall/gru4rec/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..abf198b97e6e818e1fbe59006f98492640bcee54 --- /dev/null +++ b/models/recall/gru4rec/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/recall/gru4rec/config.yaml b/models/recall/gru4rec/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35e57e01fddf030905a4ca0ae3a94ac782ea1bb3 --- /dev/null +++ b/models/recall/gru4rec/config.yaml @@ -0,0 +1,49 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +train: + trainer: + # for cluster training + strategy: "async" + + epochs: 3 + workspace: "fleetrec.models.recall.gru4rec" + + reader: + batch_size: 5 + class: "{workspace}/rsc15_reader.py" + train_data_path: "{workspace}/data/train" + + model: + models: "{workspace}/model.py" + hyper_parameters: + vocab_size: 1000 + hid_size: 100 + emb_lr_x: 10.0 + gru_lr_x: 1.0 + fc_lr_x: 1.0 + init_low_bound: -0.04 + init_high_bound: 0.04 + learning_rate: 0.01 + optimizer: adagrad + + save: + increment: + dirname: "increment" + epoch_interval: 2 + save_last: True + inference: + dirname: "inference" + epoch_interval: 4 + save_last: True diff --git a/models/recall/gru4rec/data/test/small_test.txt b/models/recall/gru4rec/data/test/small_test.txt new file mode 100644 index 0000000000000000000000000000000000000000..b4bf7189643a041b769fc88c56f6b1ec5b5229db --- /dev/null +++ b/models/recall/gru4rec/data/test/small_test.txt @@ -0,0 +1,100 @@ +0 16 +475 473 155 +491 21 +96 185 96 +29 14 13 +5 481 11 21 470 +70 5 70 11 +167 42 167 217 +72 15 73 161 172 +82 82 +97 297 97 +193 182 186 183 184 177 214 +152 152 +163 298 7 +39 73 71 +490 23 23 496 488 74 23 74 486 23 23 74 +17 17 +170 170 483 444 443 234 +25 472 +5 5 11 70 69 +149 149 455 +356 68 477 468 17 479 66 +159 172 6 71 6 6 158 13 494 169 +155 44 438 144 500 +156 9 9 +146 146 +173 10 10 461 +7 6 6 +269 48 268 +50 100 +323 174 18 +69 69 22 98 +38 171 +22 29 489 10 +0 0 +11 5 +29 13 14 232 231 451 289 452 229 +260 11 156 +166 160 166 39 +223 134 134 420 +66 401 68 132 17 84 287 5 +39 304 +65 84 132 +400 211 +145 144 +16 28 254 48 50 100 42 154 262 133 17 +0 0 +28 28 +11 476 464 +61 61 86 86 +38 38 +463 478 +437 265 +22 39 485 171 98 +434 51 344 +16 16 +67 67 67 448 +22 12 161 +15 377 147 147 374 +119 317 0 +38 484 +403 499 +432 442 +28 0 16 50 465 42 +163 487 7 162 +99 99 325 423 83 83 +154 133 +5 37 492 235 160 279 +10 10 457 493 10 460 +441 4 4 4 4 4 4 4 +153 153 +159 164 164 +328 37 +65 65 404 347 431 459 +80 80 44 44 +61 446 +162 495 7 453 +157 21 204 68 37 66 469 145 +37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 +430 445 433 +449 14 +9 9 9 9 +440 238 226 +148 148 +266 267 181 +48 498 +263 255 256 +458 158 7 +72 168 12 165 71 73 173 49 +0 0 +7 7 6 +14 29 13 6 15 14 15 13 +480 439 21 +450 21 151 +12 12 49 14 13 165 12 169 72 15 15 +91 91 +22 12 49 168 +497 101 30 411 30 482 30 53 30 101 176 415 53 447 +462 150 150 +471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 diff --git a/models/recall/gru4rec/data/train/small_train.txt b/models/recall/gru4rec/data/train/small_train.txt new file mode 100644 index 0000000000000000000000000000000000000000..6252a52c5ce3fe5bcc4f28c274e67461f47e1586 --- /dev/null +++ b/models/recall/gru4rec/data/train/small_train.txt @@ -0,0 +1,100 @@ +197 196 198 236 +93 93 384 362 363 43 +336 364 407 +421 322 +314 388 +128 58 +138 138 +46 46 46 +34 34 57 57 57 342 228 321 346 357 59 376 +110 110 +135 94 135 +27 250 27 +129 118 +18 18 18 +81 81 89 89 +27 27 +20 20 20 20 20 212 +33 33 33 33 +62 62 62 63 63 55 248 124 381 428 383 382 43 43 261 63 +90 90 78 78 +399 397 202 141 104 104 245 192 191 271 +239 332 283 88 +187 313 +136 136 324 +41 41 +352 128 +413 414 +410 45 45 45 1 1 1 1 1 1 1 1 31 31 31 31 +92 334 92 +95 285 +215 249 +390 41 +116 116 +300 252 +2 2 2 2 2 +8 8 8 8 8 8 +53 241 259 +118 129 126 94 137 208 216 299 +209 368 139 418 419 +311 180 +303 302 203 284 +369 32 32 32 32 337 +207 47 47 47 +106 107 +143 143 +179 178 +109 109 +405 79 79 371 246 +251 417 427 +333 88 387 358 123 348 394 360 36 365 +3 3 3 3 3 +189 188 +398 425 +107 406 +281 201 141 +2 2 2 +359 54 +395 385 293 +60 60 60 121 121 233 58 58 +24 199 175 24 24 24 351 386 106 +115 294 +122 122 127 127 +35 35 +282 393 +277 140 140 343 225 123 36 36 36 221 114 114 59 59 117 117 247 367 219 258 222 301 375 350 353 111 111 +275 272 273 274 331 330 305 108 76 76 108 +26 26 26 408 26 +290 18 210 291 +372 139 424 113 +341 340 335 +120 370 +224 200 +426 416 +137 319 +402 55 +54 54 +327 119 +125 125 +391 396 354 355 389 +142 142 +295 320 +113 366 +253 85 85 +56 56 310 309 308 307 278 25 25 19 19 3 312 19 19 19 3 25 +220 338 +34 130 +130 120 380 315 +339 422 +379 378 +95 56 392 115 +55 124 +126 34 +349 373 361 +195 194 +75 75 +64 64 64 +35 35 +40 40 40 242 77 244 77 243 +257 316 +103 306 102 51 52 103 105 52 52 292 318 112 286 345 237 276 112 51 102 105 diff --git a/models/recall/gru4rec/hdfs.log b/models/recall/gru4rec/hdfs.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/recall/gru4rec/model.py b/models/recall/gru4rec/model.py new file mode 100644 index 0000000000000000000000000000000000000000..5b461d76b17bae5535bc945c1d4c9e988eb7e1a8 --- /dev/null +++ b/models/recall/gru4rec/model.py @@ -0,0 +1,87 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import paddle.fluid as fluid + +from fleetrec.core.utils import envs +from fleetrec.core.model import Model as ModelBase + + +class Model(ModelBase): + def __init__(self, config): + ModelBase.__init__(self, config) + + def all_vocab_network(self): + """ network definition """ + recall_k = envs.get_global_env("hyper_parameters.recall_k", None, self._namespace) + vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace) + hid_size = envs.get_global_env("hyper_parameters.hid_size", None, self._namespace) + init_low_bound = envs.get_global_env("hyper_parameters.init_low_bound", None, self._namespace) + init_high_bound = envs.get_global_env("hyper_parameters.init_high_bound", None, self._namespace) + emb_lr_x = envs.get_global_env("hyper_parameters.emb_lr_x", None, self._namespace) + gru_lr_x = envs.get_global_env("hyper_parameters.gru_lr_x", None, self._namespace) + fc_lr_x = envs.get_global_env("hyper_parameters.fc_lr_x", None, self._namespace) + # Input data + src_wordseq = fluid.data( + name="src_wordseq", shape=[None, 1], dtype="int64", lod_level=1) + dst_wordseq = fluid.data( + name="dst_wordseq", shape=[None, 1], dtype="int64", lod_level=1) + + emb = fluid.embedding( + input=src_wordseq, + size=[vocab_size, hid_size], + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=init_low_bound, high=init_high_bound), + learning_rate=emb_lr_x), + is_sparse=True) + fc0 = fluid.layers.fc(input=emb, + size=hid_size * 3, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=init_low_bound, high=init_high_bound), + learning_rate=gru_lr_x)) + gru_h0 = fluid.layers.dynamic_gru( + input=fc0, + size=hid_size, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=init_low_bound, high=init_high_bound), + learning_rate=gru_lr_x)) + + fc = fluid.layers.fc(input=gru_h0, + size=vocab_size, + act='softmax', + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=init_low_bound, high=init_high_bound), + learning_rate=fc_lr_x)) + cost = fluid.layers.cross_entropy(input=fc, label=dst_wordseq) + acc = fluid.layers.accuracy(input=fc, label=dst_wordseq, k=recall_k) + avg_cost = fluid.layers.mean(x=cost) + + self._data_var.append(src_wordseq) + self._data_var.append(dst_wordseq) + self._cost = avg_cost + self._metrics["cost"] = avg_cost + self._metrics["acc"] = acc + + + def train_net(self): + self.all_vocab_network() + + + def infer_net(self): + pass diff --git a/models/recall/gru4rec/rsc15_reader.py b/models/recall/gru4rec/rsc15_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..fbfc4b91c749eb0c603ac335932103e2dabe83b0 --- /dev/null +++ b/models/recall/gru4rec/rsc15_reader.py @@ -0,0 +1,42 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function + +from fleetrec.core.reader import Reader +from fleetrec.core.utils import envs + + +class TrainReader(Reader): + def init(self): + pass + + def generate_sample(self, line): + """ + Read the data line by line and process it as a dictionary + """ + + def reader(): + """ + This function needs to be implemented by the user, based on data format + """ + l = line.strip().split() + l = [w for w in l] + src_seq = l[:len(l) - 1] + src_seq = [int(e) for e in src_seq] + trg_seq = l[1:] + trg_seq = [int(e) for e in trg_seq] + feature_name = ["src_wordseq", "dst_wordseq"] + yield zip(feature_name, [src_seq] + [trg_seq]) + + return reader diff --git a/models/recall/ssr/__init__.py b/models/recall/ssr/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..abf198b97e6e818e1fbe59006f98492640bcee54 --- /dev/null +++ b/models/recall/ssr/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/recall/ssr/config.yaml b/models/recall/ssr/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..640c93b9dfa9f10cb2de4092bc7d94c45ba45b85 --- /dev/null +++ b/models/recall/ssr/config.yaml @@ -0,0 +1,45 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +train: + trainer: + # for cluster training + strategy: "async" + + epochs: 3 + workspace: "fleetrec.models.recall.ssr" + + reader: + batch_size: 5 + class: "{workspace}/ssr_reader.py" + train_data_path: "{workspace}/data/train" + + model: + models: "{workspace}/model.py" + hyper_parameters: + vocab_size: 1000 + emb_dim: 128 + hidden_size: 100 + learning_rate: 0.01 + optimizer: adagrad + + save: + increment: + dirname: "increment" + epoch_interval: 2 + save_last: True + inference: + dirname: "inference" + epoch_interval: 4 + save_last: True diff --git a/models/recall/ssr/data/test/small_test.txt b/models/recall/ssr/data/test/small_test.txt new file mode 100644 index 0000000000000000000000000000000000000000..b4bf7189643a041b769fc88c56f6b1ec5b5229db --- /dev/null +++ b/models/recall/ssr/data/test/small_test.txt @@ -0,0 +1,100 @@ +0 16 +475 473 155 +491 21 +96 185 96 +29 14 13 +5 481 11 21 470 +70 5 70 11 +167 42 167 217 +72 15 73 161 172 +82 82 +97 297 97 +193 182 186 183 184 177 214 +152 152 +163 298 7 +39 73 71 +490 23 23 496 488 74 23 74 486 23 23 74 +17 17 +170 170 483 444 443 234 +25 472 +5 5 11 70 69 +149 149 455 +356 68 477 468 17 479 66 +159 172 6 71 6 6 158 13 494 169 +155 44 438 144 500 +156 9 9 +146 146 +173 10 10 461 +7 6 6 +269 48 268 +50 100 +323 174 18 +69 69 22 98 +38 171 +22 29 489 10 +0 0 +11 5 +29 13 14 232 231 451 289 452 229 +260 11 156 +166 160 166 39 +223 134 134 420 +66 401 68 132 17 84 287 5 +39 304 +65 84 132 +400 211 +145 144 +16 28 254 48 50 100 42 154 262 133 17 +0 0 +28 28 +11 476 464 +61 61 86 86 +38 38 +463 478 +437 265 +22 39 485 171 98 +434 51 344 +16 16 +67 67 67 448 +22 12 161 +15 377 147 147 374 +119 317 0 +38 484 +403 499 +432 442 +28 0 16 50 465 42 +163 487 7 162 +99 99 325 423 83 83 +154 133 +5 37 492 235 160 279 +10 10 457 493 10 460 +441 4 4 4 4 4 4 4 +153 153 +159 164 164 +328 37 +65 65 404 347 431 459 +80 80 44 44 +61 446 +162 495 7 453 +157 21 204 68 37 66 469 145 +37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 +430 445 433 +449 14 +9 9 9 9 +440 238 226 +148 148 +266 267 181 +48 498 +263 255 256 +458 158 7 +72 168 12 165 71 73 173 49 +0 0 +7 7 6 +14 29 13 6 15 14 15 13 +480 439 21 +450 21 151 +12 12 49 14 13 165 12 169 72 15 15 +91 91 +22 12 49 168 +497 101 30 411 30 482 30 53 30 101 176 415 53 447 +462 150 150 +471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 diff --git a/models/recall/ssr/data/train/small_train.txt b/models/recall/ssr/data/train/small_train.txt new file mode 100644 index 0000000000000000000000000000000000000000..6252a52c5ce3fe5bcc4f28c274e67461f47e1586 --- /dev/null +++ b/models/recall/ssr/data/train/small_train.txt @@ -0,0 +1,100 @@ +197 196 198 236 +93 93 384 362 363 43 +336 364 407 +421 322 +314 388 +128 58 +138 138 +46 46 46 +34 34 57 57 57 342 228 321 346 357 59 376 +110 110 +135 94 135 +27 250 27 +129 118 +18 18 18 +81 81 89 89 +27 27 +20 20 20 20 20 212 +33 33 33 33 +62 62 62 63 63 55 248 124 381 428 383 382 43 43 261 63 +90 90 78 78 +399 397 202 141 104 104 245 192 191 271 +239 332 283 88 +187 313 +136 136 324 +41 41 +352 128 +413 414 +410 45 45 45 1 1 1 1 1 1 1 1 31 31 31 31 +92 334 92 +95 285 +215 249 +390 41 +116 116 +300 252 +2 2 2 2 2 +8 8 8 8 8 8 +53 241 259 +118 129 126 94 137 208 216 299 +209 368 139 418 419 +311 180 +303 302 203 284 +369 32 32 32 32 337 +207 47 47 47 +106 107 +143 143 +179 178 +109 109 +405 79 79 371 246 +251 417 427 +333 88 387 358 123 348 394 360 36 365 +3 3 3 3 3 +189 188 +398 425 +107 406 +281 201 141 +2 2 2 +359 54 +395 385 293 +60 60 60 121 121 233 58 58 +24 199 175 24 24 24 351 386 106 +115 294 +122 122 127 127 +35 35 +282 393 +277 140 140 343 225 123 36 36 36 221 114 114 59 59 117 117 247 367 219 258 222 301 375 350 353 111 111 +275 272 273 274 331 330 305 108 76 76 108 +26 26 26 408 26 +290 18 210 291 +372 139 424 113 +341 340 335 +120 370 +224 200 +426 416 +137 319 +402 55 +54 54 +327 119 +125 125 +391 396 354 355 389 +142 142 +295 320 +113 366 +253 85 85 +56 56 310 309 308 307 278 25 25 19 19 3 312 19 19 19 3 25 +220 338 +34 130 +130 120 380 315 +339 422 +379 378 +95 56 392 115 +55 124 +126 34 +349 373 361 +195 194 +75 75 +64 64 64 +35 35 +40 40 40 242 77 244 77 243 +257 316 +103 306 102 51 52 103 105 52 52 292 318 112 286 345 237 276 112 51 102 105 diff --git a/models/recall/ssr/model.py b/models/recall/ssr/model.py new file mode 100644 index 0000000000000000000000000000000000000000..5fd94cf8763d79e4cc7edd92019aae1e8bf716a2 --- /dev/null +++ b/models/recall/ssr/model.py @@ -0,0 +1,138 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import paddle.fluid as fluid + +from fleetrec.core.utils import envs +from fleetrec.core.model import Model as ModelBase +import paddle.fluid.layers.tensor as tensor +import paddle.fluid.layers.io as io +import paddle.fluid.layers.control_flow as cf + + + +class BowEncoder(object): + """ bow-encoder """ + + def __init__(self): + self.param_name = "" + + def forward(self, emb): + return fluid.layers.sequence_pool(input=emb, pool_type='sum') + + +class GrnnEncoder(object): + """ grnn-encoder """ + + def __init__(self, param_name="grnn", hidden_size=128): + self.param_name = param_name + self.hidden_size = hidden_size + + def forward(self, emb): + fc0 = fluid.layers.fc(input=emb, + size=self.hidden_size * 3, + param_attr=self.param_name + "_fc.w", + bias_attr=False) + + gru_h = fluid.layers.dynamic_gru( + input=fc0, + size=self.hidden_size, + is_reverse=False, + param_attr=self.param_name + ".param", + bias_attr=self.param_name + ".bias") + return fluid.layers.sequence_pool(input=gru_h, pool_type='max') + +class PairwiseHingeLoss(object): + def __init__(self, margin=0.8): + self.margin = margin + + def forward(self, pos, neg): + loss_part1 = fluid.layers.elementwise_sub( + tensor.fill_constant_batch_size_like( + input=pos, shape=[-1, 1], value=self.margin, dtype='float32'), + pos) + loss_part2 = fluid.layers.elementwise_add(loss_part1, neg) + loss_part3 = fluid.layers.elementwise_max( + tensor.fill_constant_batch_size_like( + input=loss_part2, shape=[-1, 1], value=0.0, dtype='float32'), + loss_part2) + return loss_part3 + +class Model(ModelBase): + def __init__(self, config): + ModelBase.__init__(self, config) + + def get_correct(self, x, y): + less = tensor.cast(cf.less_than(x, y), dtype='float32') + correct = fluid.layers.reduce_sum(less) + return correct + + def train(self): + + vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace) + emb_dim = envs.get_global_env("hyper_parameters.emb_dim", None, self._namespace) + hidden_size = envs.get_global_env("hyper_parameters.hidden_size", None, self._namespace) + emb_shape = [vocab_size, emb_dim] + + self.user_encoder = GrnnEncoder() + self.item_encoder = BowEncoder() + self.pairwise_hinge_loss = PairwiseHingeLoss() + + user_data = fluid.data( + name="user", shape=[None, 1], dtype="int64", lod_level=1) + pos_item_data = fluid.data( + name="p_item", shape=[None, 1], dtype="int64", lod_level=1) + neg_item_data = fluid.data( + name="n_item", shape=[None, 1], dtype="int64", lod_level=1) + self._data_var.extend([user_data, pos_item_data, neg_item_data]) + + user_emb = fluid.embedding( + input=user_data, size=emb_shape, param_attr="emb.item") + pos_item_emb = fluid.embedding( + input=pos_item_data, size=emb_shape, param_attr="emb.item") + neg_item_emb = fluid.embedding( + input=neg_item_data, size=emb_shape, param_attr="emb.item") + user_enc = self.user_encoder.forward(user_emb) + pos_item_enc = self.item_encoder.forward(pos_item_emb) + neg_item_enc = self.item_encoder.forward(neg_item_emb) + user_hid = fluid.layers.fc(input=user_enc, + size=hidden_size, + param_attr='user.w', + bias_attr="user.b") + pos_item_hid = fluid.layers.fc(input=pos_item_enc, + size=hidden_size, + param_attr='item.w', + bias_attr="item.b") + neg_item_hid = fluid.layers.fc(input=neg_item_enc, + size=hidden_size, + param_attr='item.w', + bias_attr="item.b") + cos_pos = fluid.layers.cos_sim(user_hid, pos_item_hid) + cos_neg = fluid.layers.cos_sim(user_hid, neg_item_hid) + hinge_loss = self.pairwise_hinge_loss.forward(cos_pos, cos_neg) + avg_cost = fluid.layers.mean(hinge_loss) + correct = self.get_correct(cos_neg, cos_pos) + + self._cost = avg_cost + self._metrics["correct"] = correct + self._metrics["hinge_loss"] = hinge_loss + + + def train_net(self): + self.train() + + + def infer_net(self): + pass diff --git a/models/recall/ssr/ssr_reader.py b/models/recall/ssr/ssr_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..ba81e01f7b84a462938383a141eeb2bfe3f816d1 --- /dev/null +++ b/models/recall/ssr/ssr_reader.py @@ -0,0 +1,47 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function + +from fleetrec.core.reader import Reader +from fleetrec.core.utils import envs +import random + + +class TrainReader(Reader): + def init(self): + pass + + def sample_neg_from_seq(self, seq): + return seq[random.randint(0, len(seq) - 1)] + + + def generate_sample(self, line): + """ + Read the data line by line and process it as a dictionary + """ + + def reader(): + """ + This function needs to be implemented by the user, based on data format + """ + ids = line.strip().split() + conv_ids = [int(i) for i in ids] + boundary = len(ids) - 1 + src = conv_ids[:boundary] + pos_tgt = [conv_ids[boundary]] + neg_tgt = [self.sample_neg_from_seq(src)] + feature_name = ["user", "p_item", "n_item"] + yield zip(feature_name, [src] + [pos_tgt] + [neg_tgt]) + + return reader