Merge branch 'develop' into 'develop'

add multitask ssr gru4rec

See merge request !18

models/multitask/esmm/__init__.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

models/multitask/esmm/config.yaml

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+train:
+  trainer:
+    # for cluster training
+    strategy: "async"
+
+  epochs: 3
+  workspace: "fleetrec.models.multitask.esmm"
+
+  reader:
+    batch_size: 2
+    class: "{workspace}/esmm_reader.py"
+    train_data_path: "{workspace}/data/train"
+
+  model:
+    models: "{workspace}/model.py"
+    hyper_parameters:
+      vocab_size: 10000
+      embed_size: 128
+      learning_rate: 0.001
+      optimizer: adam
+
+  save:
+    increment:
+      dirname: "increment"
+      epoch_interval: 2
+      save_last: True
+    inference:
+      dirname: "inference"
+      epoch_interval: 4
+      save_last: True

models/multitask/esmm/data/test/small_test.txt

+0 16 
+475 473 155 
+491 21 
+96 185 96 
+29 14 13 
+5 481 11 21 470 
+70 5 70 11 
+167 42 167 217 
+72 15 73 161 172 
+82 82 
+97 297 97 
+193 182 186 183 184 177 214 
+152 152 
+163 298 7 
+39 73 71 
+490 23 23 496 488 74 23 74 486 23 23 74 
+17 17 
+170 170 483 444 443 234 
+25 472 
+5 5 11 70 69 
+149 149 455 
+356 68 477 468 17 479 66 
+159 172 6 71 6 6 158 13 494 169 
+155 44 438 144 500 
+156 9 9 
+146 146 
+173 10 10 461 
+7 6 6 
+269 48 268 
+50 100 
+323 174 18 
+69 69 22 98 
+38 171 
+22 29 489 10 
+0 0 
+11 5 
+29 13 14 232 231 451 289 452 229 
+260 11 156 
+166 160 166 39 
+223 134 134 420 
+66 401 68 132 17 84 287 5 
+39 304 
+65 84 132 
+400 211 
+145 144 
+16 28 254 48 50 100 42 154 262 133 17 
+0 0 
+28 28 
+11 476 464 
+61 61 86 86 
+38 38 
+463 478 
+437 265 
+22 39 485 171 98 
+434 51 344 
+16 16 
+67 67 67 448 
+22 12 161 
+15 377 147 147 374 
+119 317 0 
+38 484 
+403 499 
+432 442 
+28 0 16 50 465 42 
+163 487 7 162 
+99 99 325 423 83 83 
+154 133 
+5 37 492 235 160 279 
+10 10 457 493 10 460 
+441 4 4 4 4 4 4 4 
+153 153 
+159 164 164 
+328 37 
+65 65 404 347 431 459 
+80 80 44 44 
+61 446 
+162 495 7 453 
+157 21 204 68 37 66 469 145 
+37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 
+430 445 433 
+449 14 
+9 9 9 9 
+440 238 226 
+148 148 
+266 267 181 
+48 498 
+263 255 256 
+458 158 7 
+72 168 12 165 71 73 173 49 
+0 0 
+7 7 6 
+14 29 13 6 15 14 15 13 
+480 439 21 
+450 21 151 
+12 12 49 14 13 165 12 169 72 15 15 
+91 91 
+22 12 49 168 
+497 101 30 411 30 482 30 53 30 101 176 415 53 447 
+462 150 150 
+471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 

models/multitask/esmm/data/train/small.csv

+1,0,0,210:1,210:2,210:3,216:4,301:5,205:6,206:7,207:8,508:9,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876
+2,0,0,210:877,210:878,210:879,210:880,210:881,216:882,301:5,205:883,206:884,207:885,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876
+3,1,0,210:886,210:887,509:888,508:889,702:890,853:891,207:892,206:893,205:894,301:5,216:895,210:896,210:897,210:898,210:899,210:900,210:901,210:902,210:903,210:904,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876
+4,0,0,301:5,210:905,210:906,210:907,210:908,508:909,853:910,853:911,853:912,207:913,206:914,205:915,216:916,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876
+5,0,0,205:917,301:5,216:918,210:919,210:920,210:921,508:922,207:923,206:924,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876
+6,0,0,210:897,210:925,210:926,210:927,210:928,210:929,210:930,210:931,210:901,210:932,210:933,210:934,210:935,210:936,216:937,301:5,205:938,206:893,207:939,853:891,508:889,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876
+7,0,0,210:887,301:5,205:940,206:893,207:941,853:891,508:889,210:901,210:925,210:897,210:931,210:929,216:937,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876
+8,0,0,216:942,210:943,210:944,509:945,508:946,702:947,207:948,206:949,205:950,301:5,210:951,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876
+9,0,0,206:952,205:953,301:5,216:954,210:955,210:956,210:957,509:958,207:959,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876
+10,0,0,216:942,210:960,210:961,210:962,508:963,702:947,207:964,206:965,205:966,301:5,121:10,122:11,101:12,124:13,125:14,127:15,128:16,129:17,150_14:18,150_14:19,150_14:20,150_14:21,150_14:22,150_14:23,150_14:24,150_14:25,150_14:26,150_14:27,150_14:28,150_14:29,150_14:30,150_14:31,150_14:32,150_14:33,150_14:34,150_14:35,150_14:36,150_14:37,150_14:38,150_14:39,150_14:40,150_14:41,150_14:42,150_14:43,150_14:44,150_14:45,150_14:46,150_14:47,150_14:48,150_14:49,150_14:50,150_14:51,150_14:52,150_14:53,150_14:54,150_14:55,150_14:56,150_14:57,150_14:58,150_14:59,150_14:60,150_14:61,150_14:62,150_14:63,150_14:64,150_14:65,150_14:66,150_14:67,150_14:68,150_14:69,150_14:70,150_14:71,150_14:72,150_14:73,150_14:74,150_14:75,150_14:76,150_14:77,150_14:78,150_14:79,150_14:80,150_14:81,150_14:82,150_14:83,150_14:84,150_14:85,150_14:86,150_14:87,150_14:88,150_14:89,150_14:90,150_14:91,150_14:92,150_14:93,150_14:94,150_14:95,150_14:96,150_14:97,150_14:98,150_14:99,150_14:100,150_14:101,150_14:102,150_14:103,150_14:104,150_14:105,150_14:106,150_14:107,150_14:108,150_14:109,150_14:110,150_14:111,150_14:112,150_14:113,150_14:114,150_14:115,150_14:116,150_14:117,150_14:118,150_14:119,150_14:120,150_14:121,150_14:122,150_14:123,150_14:124,150_14:125,150_14:126,150_14:127,150_14:128,150_14:129,150_14:130,150_14:131,150_14:132,150_14:133,150_14:134,150_14:135,150_14:136,150_14:137,150_14:138,150_14:139,150_14:140,150_14:141,150_14:142,150_14:143,150_14:144,150_14:145,150_14:146,150_14:147,150_14:148,150_14:149,150_14:150,150_14:151,150_14:152,150_14:153,150_14:154,150_14:155,150_14:156,150_14:157,150_14:158,150_14:159,150_14:160,150_14:161,150_14:162,150_14:163,150_14:164,150_14:165,150_14:166,150_14:167,150_14:168,150_14:169,150_14:170,150_14:171,150_14:172,150_14:173,150_14:174,150_14:175,150_14:176,150_14:177,150_14:178,150_14:179,150_14:180,150_14:181,150_14:182,150_14:183,150_14:184,150_14:185,150_14:186,150_14:187,150_14:188,150_14:189,150_14:190,150_14:191,150_14:192,150_14:193,150_14:194,150_14:195,150_14:196,150_14:197,150_14:198,150_14:199,150_14:200,150_14:201,150_14:202,150_14:203,150_14:204,150_14:205,150_14:206,150_14:207,150_14:208,127_14:209,127_14:210,127_14:211,127_14:212,127_14:213,127_14:214,127_14:215,127_14:216,127_14:217,127_14:218,127_14:219,127_14:220,127_14:221,127_14:222,127_14:223,127_14:224,127_14:225,127_14:226,127_14:227,127_14:228,127_14:229,127_14:230,127_14:231,127_14:232,127_14:233,127_14:234,127_14:235,127_14:236,127_14:237,127_14:238,127_14:239,127_14:240,127_14:241,127_14:242,127_14:243,127_14:244,127_14:245,127_14:246,127_14:247,127_14:248,127_14:249,127_14:250,127_14:251,127_14:252,127_14:253,127_14:254,127_14:255,127_14:256,127_14:257,127_14:258,127_14:259,127_14:260,127_14:261,127_14:262,127_14:263,127_14:264,127_14:265,127_14:266,127_14:267,127_14:268,127_14:269,127_14:270,127_14:271,127_14:272,127_14:273,127_14:274,127_14:275,127_14:276,127_14:277,127_14:278,127_14:279,127_14:280,127_14:281,127_14:282,127_14:283,127_14:284,127_14:285,127_14:286,127_14:287,127_14:288,127_14:289,127_14:290,127_14:291,127_14:292,127_14:293,127_14:294,127_14:295,127_14:296,127_14:297,127_14:298,127_14:299,127_14:300,127_14:301,127_14:302,127_14:303,127_14:304,127_14:305,127_14:306,127_14:307,127_14:308,127_14:309,127_14:310,127_14:311,127_14:312,127_14:313,127_14:314,127_14:315,127_14:316,127_14:317,127_14:318,127_14:319,127_14:320,127_14:321,127_14:322,127_14:323,127_14:324,127_14:325,127_14:326,127_14:327,127_14:328,127_14:329,127_14:330,127_14:331,127_14:332,127_14:333,127_14:334,127_14:335,127_14:336,127_14:337,127_14:338,127_14:339,127_14:340,127_14:341,127_14:342,127_14:343,127_14:344,127_14:345,127_14:346,127_14:347,127_14:348,127_14:349,127_14:350,127_14:351,127_14:352,127_14:353,127_14:354,127_14:355,127_14:356,127_14:357,127_14:358,127_14:359,127_14:360,127_14:361,127_14:362,127_14:363,127_14:364,127_14:365,127_14:366,127_14:367,127_14:368,127_14:369,127_14:370,127_14:371,127_14:372,127_14:373,127_14:374,127_14:375,127_14:376,127_14:377,127_14:378,127_14:379,127_14:380,127_14:381,127_14:382,127_14:383,127_14:384,127_14:385,127_14:386,127_14:387,127_14:388,127_14:389,127_14:390,127_14:391,127_14:392,127_14:393,127_14:394,127_14:395,127_14:396,127_14:397,127_14:398,127_14:399,127_14:400,127_14:401,127_14:402,127_14:403,127_14:404,127_14:405,127_14:406,127_14:407,127_14:408,127_14:409,127_14:410,127_14:411,127_14:412,127_14:413,127_14:414,127_14:415,127_14:416,127_14:417,109_14:418,109_14:419,109_14:420,109_14:421,109_14:422,109_14:423,109_14:424,109_14:425,109_14:426,109_14:427,109_14:428,109_14:429,109_14:430,109_14:431,109_14:432,109_14:433,109_14:434,109_14:435,109_14:436,109_14:437,109_14:438,109_14:439,109_14:440,109_14:441,109_14:442,109_14:443,109_14:444,109_14:445,109_14:446,109_14:447,109_14:448,109_14:449,109_14:450,109_14:451,109_14:452,109_14:453,109_14:454,109_14:455,109_14:456,109_14:457,109_14:458,109_14:459,109_14:460,109_14:461,109_14:462,109_14:463,109_14:464,109_14:465,109_14:466,109_14:467,109_14:468,109_14:469,109_14:470,109_14:471,109_14:472,109_14:473,109_14:474,109_14:475,109_14:476,109_14:477,109_14:478,109_14:479,109_14:480,109_14:481,109_14:482,109_14:483,109_14:484,109_14:485,109_14:486,109_14:487,109_14:488,109_14:489,109_14:490,109_14:491,109_14:492,109_14:493,109_14:494,109_14:495,109_14:496,109_14:497,109_14:498,109_14:499,109_14:500,109_14:501,109_14:502,109_14:503,109_14:504,109_14:505,109_14:506,109_14:507,109_14:508,109_14:509,109_14:510,109_14:511,109_14:512,109_14:513,109_14:514,109_14:515,109_14:516,109_14:517,109_14:518,109_14:519,109_14:520,109_14:521,109_14:522,109_14:523,109_14:524,109_14:525,109_14:526,109_14:527,109_14:528,109_14:529,109_14:530,109_14:531,109_14:532,109_14:533,109_14:534,110_14:535,110_14:536,110_14:537,110_14:538,110_14:539,110_14:540,110_14:541,110_14:542,110_14:543,110_14:544,110_14:545,110_14:546,110_14:547,110_14:548,110_14:549,110_14:550,110_14:551,110_14:552,110_14:553,110_14:554,110_14:555,110_14:556,110_14:557,110_14:558,110_14:559,110_14:560,110_14:561,110_14:562,110_14:563,110_14:564,110_14:565,110_14:566,110_14:567,110_14:568,110_14:569,110_14:570,110_14:571,110_14:572,110_14:573,110_14:574,110_14:575,110_14:576,110_14:577,110_14:578,110_14:579,110_14:580,110_14:581,110_14:582,110_14:583,110_14:584,110_14:585,110_14:586,110_14:587,110_14:588,110_14:589,110_14:590,110_14:591,110_14:592,110_14:593,110_14:594,110_14:595,110_14:596,110_14:597,110_14:598,110_14:599,110_14:600,110_14:601,110_14:602,110_14:603,110_14:604,110_14:605,110_14:606,110_14:607,110_14:608,110_14:609,110_14:610,110_14:611,110_14:612,110_14:613,110_14:614,110_14:615,110_14:616,110_14:617,110_14:618,110_14:619,110_14:620,110_14:621,110_14:622,110_14:623,110_14:624,110_14:625,110_14:626,110_14:627,110_14:628,110_14:629,110_14:630,110_14:631,110_14:632,110_14:633,110_14:634,110_14:635,110_14:636,110_14:637,110_14:638,110_14:639,110_14:640,110_14:641,110_14:642,110_14:643,110_14:644,110_14:645,110_14:646,110_14:647,110_14:648,110_14:649,110_14:650,110_14:651,110_14:652,110_14:653,110_14:654,110_14:655,110_14:656,110_14:657,110_14:658,110_14:659,110_14:660,110_14:661,110_14:662,110_14:663,110_14:664,110_14:665,110_14:666,110_14:667,110_14:668,110_14:669,110_14:670,110_14:671,110_14:672,110_14:673,110_14:674,110_14:675,110_14:676,110_14:677,110_14:678,110_14:679,110_14:680,110_14:681,110_14:682,110_14:683,110_14:684,110_14:685,110_14:686,110_14:687,110_14:688,110_14:689,110_14:690,110_14:691,110_14:692,110_14:693,110_14:694,110_14:695,110_14:696,110_14:697,110_14:698,110_14:699,110_14:700,110_14:701,110_14:702,110_14:703,110_14:704,110_14:705,110_14:706,110_14:707,110_14:708,110_14:709,110_14:710,110_14:711,110_14:712,110_14:713,110_14:714,110_14:715,110_14:716,110_14:717,110_14:718,110_14:719,110_14:720,110_14:721,110_14:722,110_14:723,110_14:724,110_14:725,110_14:726,110_14:727,110_14:728,110_14:729,110_14:730,110_14:731,110_14:732,110_14:733,110_14:734,110_14:735,110_14:736,110_14:737,110_14:738,110_14:739,110_14:740,110_14:741,110_14:742,110_14:743,110_14:744,110_14:745,110_14:746,110_14:747,110_14:748,110_14:749,110_14:750,110_14:751,110_14:752,110_14:753,110_14:754,110_14:755,110_14:756,110_14:757,110_14:758,110_14:759,110_14:760,110_14:761,110_14:762,110_14:763,110_14:764,110_14:765,110_14:766,110_14:767,110_14:768,110_14:769,110_14:770,110_14:771,110_14:772,110_14:773,110_14:774,110_14:775,110_14:776,110_14:777,110_14:778,110_14:779,110_14:780,110_14:781,110_14:782,110_14:783,110_14:784,110_14:785,110_14:786,110_14:787,110_14:788,110_14:789,110_14:790,110_14:791,110_14:792,110_14:793,110_14:794,110_14:795,110_14:796,110_14:797,110_14:798,110_14:799,110_14:800,110_14:801,110_14:802,110_14:803,110_14:804,110_14:805,110_14:806,110_14:807,110_14:808,110_14:809,110_14:810,110_14:811,110_14:812,110_14:813,110_14:814,110_14:815,110_14:816,110_14:817,110_14:818,110_14:819,110_14:820,110_14:821,110_14:822,110_14:823,110_14:824,110_14:825,110_14:826,110_14:827,110_14:828,110_14:829,110_14:830,110_14:831,110_14:832,110_14:833,110_14:834,110_14:835,110_14:836,110_14:837,110_14:838,110_14:839,110_14:840,110_14:841,110_14:842,110_14:843,110_14:844,110_14:845,110_14:846,110_14:847,110_14:848,110_14:849,110_14:850,110_14:851,110_14:852,110_14:853,110_14:854,110_14:855,110_14:856,110_14:857,110_14:858,110_14:859,110_14:860,110_14:861,110_14:862,110_14:863,110_14:864,110_14:865,110_14:866,110_14:867,110_14:868,110_14:869,110_14:870,110_14:871,110_14:872,110_14:873,110_14:874,110_14:875,110_14:876

models/multitask/esmm/esmm_reader.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+from fleetrec.core.reader import Reader
+from fleetrec.core.utils import envs
+from collections import defaultdict
+import numpy as np
+
+
+class TrainReader(Reader):
+    def init(self):
+        all_field_id = ['101', '109_14', '110_14', '127_14', '150_14', '121', '122', '124', '125', '126', '127', '128', '129',
+                        '205', '206', '207', '210', '216', '508', '509', '702', '853', '301']
+        self.all_field_id_dict = defaultdict(int)
+        for i,field_id in enumerate(all_field_id):
+            self.all_field_id_dict[field_id] = [False,i]
+
+    def generate_sample(self, line):
+        """
+        Read the data line by line and process it as a dictionary
+        """
+
+        def reader():
+            """
+            This function needs to be implemented by the user, based on data format
+            """
+            features = line.strip().split(',')
+            #ctr = list(map(int, features[1]))
+            #cvr = list(map(int, features[2]))
+            ctr = int(features[1])
+            cvr = int(features[2])
+            
+            padding = 0
+            output = [(field_id,[]) for field_id in self.all_field_id_dict]
+
+            for elem in features[4:]:
+                field_id,feat_id = elem.strip().split(':')
+                if field_id not in self.all_field_id_dict:
+                    continue
+                self.all_field_id_dict[field_id][0] = True
+                index = self.all_field_id_dict[field_id][1]
+                #feat_id = list(map(int, feat_id))    
+                output[index][1].append(int(feat_id)) 
+                
+            for field_id in self.all_field_id_dict:
+                visited,index = self.all_field_id_dict[field_id]
+                if visited:
+                    self.all_field_id_dict[field_id][0] = False
+                else:
+                    output[index][1].append(padding) 
+            output.append(('ctr', [ctr]))
+            output.append(('cvr', [cvr]))
+            yield output
+        return reader

models/multitask/esmm/model.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import paddle.fluid as fluid
+
+from fleetrec.core.utils import envs
+from fleetrec.core.model import Model as ModelBase
+import numpy as np
+
+
+class Model(ModelBase):
+    def __init__(self, config):
+        ModelBase.__init__(self, config)
+
+    def fc(self,tag, data, out_dim, active='prelu'):
+        
+        init_stddev = 1.0
+        scales = 1.0  / np.sqrt(data.shape[1])
+        
+        p_attr = fluid.param_attr.ParamAttr(name='%s_weight' % tag,
+                    initializer=fluid.initializer.NormalInitializer(loc=0.0, scale=init_stddev * scales))
+                    
+        b_attr = fluid.ParamAttr(name='%s_bias' % tag, initializer=fluid.initializer.Constant(0.1))
+        
+        out = fluid.layers.fc(input=data,
+                            size=out_dim,
+                            act=active,
+                            param_attr=p_attr, 
+                            bias_attr =b_attr,
+                            name=tag)
+        return out
+        
+    def input_data(self):
+        sparse_input_ids = [
+            fluid.data(name="field_" + str(i), shape=[-1, 1], dtype="int64", lod_level=1) for i in range(0,23)
+        ]
+        label_ctr = fluid.data(name="ctr", shape=[-1, 1], dtype="int64")
+        label_cvr = fluid.data(name="cvr", shape=[-1, 1], dtype="int64")
+        inputs = sparse_input_ids + [label_ctr] + [label_cvr]
+        self._data_var.extend(inputs)
+        
+        return inputs
+    
+    def net(self, inputs):
+        
+        vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace)
+        embed_size = envs.get_global_env("hyper_parameters.embed_size", None, self._namespace)
+        emb = []
+        for data in inputs[0:-2]:
+            feat_emb = fluid.embedding(input=data,
+                                size=[vocab_size, embed_size],
+                                param_attr=fluid.ParamAttr(name='dis_emb',
+                                                            learning_rate=5,
+                                                            initializer=fluid.initializer.Xavier(fan_in=embed_size,fan_out=embed_size)
+                                                            ),
+                                is_sparse=True)
+            field_emb = fluid.layers.sequence_pool(input=feat_emb,pool_type='sum')
+            emb.append(field_emb)
+        concat_emb = fluid.layers.concat(emb, axis=1)
+        
+        # ctr
+        active = 'relu'
+        ctr_fc1 = self.fc('ctr_fc1', concat_emb, 200, active)
+        ctr_fc2 = self.fc('ctr_fc2', ctr_fc1, 80, active)
+        ctr_out = self.fc('ctr_out', ctr_fc2, 2, 'softmax')
+        
+        # cvr
+        cvr_fc1 = self.fc('cvr_fc1', concat_emb, 200, active)
+        cvr_fc2 = self.fc('cvr_fc2', cvr_fc1, 80, active)
+        cvr_out = self.fc('cvr_out', cvr_fc2, 2,'softmax')
+    
+        ctr_clk = inputs[-2]
+        ctcvr_buy = inputs[-1]
+        
+        ctr_prop_one = fluid.layers.slice(ctr_out, axes=[1], starts=[1], ends=[2])
+        cvr_prop_one = fluid.layers.slice(cvr_out, axes=[1], starts=[1], ends=[2])
+        
+        ctcvr_prop_one = fluid.layers.elementwise_mul(ctr_prop_one, cvr_prop_one)
+        ctcvr_prop = fluid.layers.concat(input=[1-ctcvr_prop_one,ctcvr_prop_one], axis = 1)
+    
+        loss_ctr = fluid.layers.cross_entropy(input=ctr_out, label=ctr_clk)
+        loss_ctcvr = fluid.layers.cross_entropy(input=ctcvr_prop, label=ctcvr_buy)
+        cost = loss_ctr + loss_ctcvr
+        avg_cost = fluid.layers.mean(cost)
+
+        auc_ctr, batch_auc_ctr, auc_states_ctr = fluid.layers.auc(input=ctr_out, label=ctr_clk)
+        auc_ctcvr, batch_auc_ctcvr, auc_states_ctcvr = fluid.layers.auc(input=ctcvr_prop, label=ctcvr_buy)
+    
+        self._cost = avg_cost
+        self._metrics["AUC_ctr"] = auc_ctr
+        self._metrics["BATCH_AUC_ctr"] = batch_auc_ctr
+        self._metrics["AUC_ctcvr"] = auc_ctcvr
+        self._metrics["BATCH_AUC_ctcvr"] = batch_auc_ctcvr
+
+
+    def train_net(self):
+        input_data = self.input_data()
+        self.net(input_data)
+
+
+    def infer_net(self):
+        pass

models/multitask/mmoe/__init__.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

models/multitask/mmoe/census_reader.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+from fleetrec.core.reader import Reader
+from fleetrec.core.utils import envs
+import numpy as np
+
+
+class TrainReader(Reader):
+    def init(self):
+        pass
+
+    def generate_sample(self, line):
+        """
+        Read the data line by line and process it as a dictionary
+        """
+
+        def reader():
+            """
+            This function needs to be implemented by the user, based on data format
+            """
+            l = line.strip().split(',')
+            l = list(map(float, l))
+            label_income = []
+            label_marital = []
+            data = l[2:]
+            if int(l[1]) == 0:
+                label_income = [1, 0]
+            elif int(l[1]) == 1:
+                label_income = [0, 1]
+            if int(l[0]) == 0:
+                label_marital = [1, 0]
+            elif int(l[0]) == 1:
+                label_marital = [0, 1]
+            #label_income = np.array(label_income)
+            #label_marital = np.array(label_marital)
+            feature_name = ["input", "label_income", "label_marital"]
+            yield zip(feature_name, [data] + [label_income] + [label_marital])
+
+        return reader

models/multitask/mmoe/config.yaml

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+train:
+  trainer:
+    # for cluster training
+    strategy: "async"
+
+  epochs: 3
+  workspace: "fleetrec.models.multitask.mmoe"
+
+  reader:
+    batch_size: 2
+    class: "{workspace}/census_reader.py"
+    train_data_path: "{workspace}/data/train"
+
+  model:
+    models: "{workspace}/model.py"
+    hyper_parameters:
+      feature_size: 499
+      expert_num: 8
+      gate_num: 2
+      expert_size: 16
+      tower_size: 8
+      learning_rate: 0.001
+      optimizer: adam
+
+  save:
+    increment:
+      dirname: "increment"
+      epoch_interval: 2
+      save_last: True
+    inference:
+      dirname: "inference"
+      epoch_interval: 4
+      save_last: True

models/multitask/mmoe/data/test/small_test.txt

+0 16 
+475 473 155 
+491 21 
+96 185 96 
+29 14 13 
+5 481 11 21 470 
+70 5 70 11 
+167 42 167 217 
+72 15 73 161 172 
+82 82 
+97 297 97 
+193 182 186 183 184 177 214 
+152 152 
+163 298 7 
+39 73 71 
+490 23 23 496 488 74 23 74 486 23 23 74 
+17 17 
+170 170 483 444 443 234 
+25 472 
+5 5 11 70 69 
+149 149 455 
+356 68 477 468 17 479 66 
+159 172 6 71 6 6 158 13 494 169 
+155 44 438 144 500 
+156 9 9 
+146 146 
+173 10 10 461 
+7 6 6 
+269 48 268 
+50 100 
+323 174 18 
+69 69 22 98 
+38 171 
+22 29 489 10 
+0 0 
+11 5 
+29 13 14 232 231 451 289 452 229 
+260 11 156 
+166 160 166 39 
+223 134 134 420 
+66 401 68 132 17 84 287 5 
+39 304 
+65 84 132 
+400 211 
+145 144 
+16 28 254 48 50 100 42 154 262 133 17 
+0 0 
+28 28 
+11 476 464 
+61 61 86 86 
+38 38 
+463 478 
+437 265 
+22 39 485 171 98 
+434 51 344 
+16 16 
+67 67 67 448 
+22 12 161 
+15 377 147 147 374 
+119 317 0 
+38 484 
+403 499 
+432 442 
+28 0 16 50 465 42 
+163 487 7 162 
+99 99 325 423 83 83 
+154 133 
+5 37 492 235 160 279 
+10 10 457 493 10 460 
+441 4 4 4 4 4 4 4 
+153 153 
+159 164 164 
+328 37 
+65 65 404 347 431 459 
+80 80 44 44 
+61 446 
+162 495 7 453 
+157 21 204 68 37 66 469 145 
+37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 
+430 445 433 
+449 14 
+9 9 9 9 
+440 238 226 
+148 148 
+266 267 181 
+48 498 
+263 255 256 
+458 158 7 
+72 168 12 165 71 73 173 49 
+0 0 
+7 7 6 
+14 29 13 6 15 14 15 13 
+480 439 21 
+450 21 151 
+12 12 49 14 13 165 12 169 72 15 15 
+91 91 
+22 12 49 168 
+497 101 30 411 30 482 30 53 30 101 176 415 53 447 
+462 150 150 
+471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 

models/multitask/mmoe/data/train/train_data

+0,0,73,0,0,0,0,1700.09,0,0,2,0,95,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,58,0,0,0,0,1053.55,1,0,2,52,94,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+1,0,18,0,0,0,0,991.95,0,0,2,0,95,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0
+1,0,9,0,0,0,0,1758.14,0,0,0,0,94,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+1,0,10,0,0,0,0,1069.16,0,0,0,0,94,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,48,1200,0,0,0,162.61,1,2,2,52,95,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,42,0,5178,0,0,1535.86,6,0,2,52,94,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+1,0,28,0,0,0,0,898.83,4,0,2,30,95,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,47,876,0,0,0,1661.53,5,0,2,52,95,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,34,0,0,0,0,1146.79,6,0,2,52,94,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0

models/multitask/mmoe/model.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import paddle.fluid as fluid
+
+from fleetrec.core.utils import envs
+from fleetrec.core.model import Model as ModelBase
+
+
+class Model(ModelBase):
+    def __init__(self, config):
+        ModelBase.__init__(self, config)
+
+    def MMOE(self):
+
+        feature_size = envs.get_global_env("hyper_parameters.feature_size", None, self._namespace)
+        expert_num = envs.get_global_env("hyper_parameters.expert_num", None, self._namespace)
+        gate_num = envs.get_global_env("hyper_parameters.gate_num", None, self._namespace)
+        expert_size = envs.get_global_env("hyper_parameters.expert_size", None, self._namespace)
+        tower_size = envs.get_global_env("hyper_parameters.tower_size", None, self._namespace)
+
+        input_data = fluid.data(name="input", shape=[-1, feature_size], dtype="float32")
+        label_income = fluid.data(name="label_income", shape=[-1, 2], dtype="float32", lod_level=0)
+        label_marital = fluid.data(name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0)
+        
+        self._data_var.extend([input_data, label_income, label_marital])
+        # f_{i}(x) = activation(W_{i} * x + b), where activation is ReLU according to the paper
+        expert_outputs = []
+        for i in range(0, expert_num):
+            expert_output = fluid.layers.fc(input=input_data,
+                                           size=expert_size,
+                                           act='relu',
+                                           bias_attr=fluid.ParamAttr(learning_rate=1.0),
+                                           name='expert_' + str(i))
+            expert_outputs.append(expert_output)
+        expert_concat = fluid.layers.concat(expert_outputs, axis=1)
+        expert_concat = fluid.layers.reshape(expert_concat,[-1, expert_num, expert_size])
+        
+        
+        # g^{k}(x) = activation(W_{gk} * x + b), where activation is softmax according to the paper
+        output_layers = []
+        for i in range(0, gate_num):
+            cur_gate = fluid.layers.fc(input=input_data,
+                                       size=expert_num,
+                                       act='softmax',
+                                       bias_attr=fluid.ParamAttr(learning_rate=1.0),
+                                       name='gate_' + str(i))
+            # f^{k}(x) = sum_{i=1}^{n}(g^{k}(x)_{i} * f_{i}(x))
+            cur_gate_expert = fluid.layers.elementwise_mul(expert_concat, cur_gate, axis=0)  
+            cur_gate_expert = fluid.layers.reduce_sum(cur_gate_expert, dim=1)
+            # Build tower layer
+            cur_tower =  fluid.layers.fc(input=cur_gate_expert,
+                                      size=tower_size,
+                                      act='relu',
+                                      name='task_layer_' + str(i))  
+            out =  fluid.layers.fc(input=cur_tower,
+                                   size=2,
+                                   act='softmax',
+                                   name='out_' + str(i))
+                
+            output_layers.append(out)
+
+        pred_income = fluid.layers.clip(output_layers[0], min=1e-15, max=1.0 - 1e-15)
+        pred_marital = fluid.layers.clip(output_layers[1], min=1e-15, max=1.0 - 1e-15)
+
+        cost_income = fluid.layers.cross_entropy(input=pred_income, label=label_income,soft_label = True)
+        cost_marital = fluid.layers.cross_entropy(input=pred_marital, label=label_marital,soft_label = True)
+        
+        label_income_1 = fluid.layers.slice(label_income, axes=[1], starts=[1], ends=[2])
+        label_marital_1 = fluid.layers.slice(label_marital, axes=[1], starts=[1], ends=[2])
+        
+        auc_income, batch_auc_1, auc_states_1  = fluid.layers.auc(input=pred_income, label=fluid.layers.cast(x=label_income_1, dtype='int64'))
+        auc_marital, batch_auc_2, auc_states_2 = fluid.layers.auc(input=pred_marital, label=fluid.layers.cast(x=label_marital_1, dtype='int64'))
+        
+        avg_cost_income = fluid.layers.mean(x=cost_income)
+        avg_cost_marital = fluid.layers.mean(x=cost_marital)
+        
+        cost =  avg_cost_income + avg_cost_marital
+    
+        self._cost = cost
+        self._metrics["AUC_income"] = auc_income
+        self._metrics["BATCH_AUC_income"] = batch_auc_1
+        self._metrics["AUC_marital"] = auc_marital
+        self._metrics["BATCH_AUC_marital"] = batch_auc_2
+
+
+    def train_net(self):
+        self.MMOE()
+
+
+    def infer_net(self):
+        pass

models/multitask/share-bottom/__init__.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

models/multitask/share-bottom/census_reader.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+from fleetrec.core.reader import Reader
+from fleetrec.core.utils import envs
+import numpy as np
+
+
+class TrainReader(Reader):
+    def init(self):
+        pass
+
+    def generate_sample(self, line):
+        """
+        Read the data line by line and process it as a dictionary
+        """
+
+        def reader():
+            """
+            This function needs to be implemented by the user, based on data format
+            """
+            l = line.strip().split(',')
+            l = list(map(float, l))
+            label_income = []
+            label_marital = []
+            data = l[2:]
+            if int(l[1]) == 0:
+                label_income = [1, 0]
+            elif int(l[1]) == 1:
+                label_income = [0, 1]
+            if int(l[0]) == 0:
+                label_marital = [1, 0]
+            elif int(l[0]) == 1:
+                label_marital = [0, 1]
+            #label_income = np.array(label_income)
+            #label_marital = np.array(label_marital)
+            feature_name = ["input", "label_income", "label_marital"]
+            yield zip(feature_name, [data] + [label_income] + [label_marital])
+
+        return reader

models/multitask/share-bottom/config.yaml

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+train:
+  trainer:
+    # for cluster training
+    strategy: "async"
+
+  epochs: 3
+  workspace: "fleetrec.models.multitask.share-bottom"
+
+  reader:
+    batch_size: 2
+    class: "{workspace}/census_reader.py"
+    train_data_path: "{workspace}/data/train"
+
+  model:
+    models: "{workspace}/model.py"
+    hyper_parameters:
+      feature_size: 499
+      bottom_size: 117
+      tower_nums: 2
+      tower_size: 8
+      learning_rate: 0.001
+      optimizer: adam
+
+  save:
+    increment:
+      dirname: "increment"
+      epoch_interval: 2
+      save_last: True
+    inference:
+      dirname: "inference"
+      epoch_interval: 4
+      save_last: True

models/multitask/share-bottom/data/test/small_test.txt

+0 16 
+475 473 155 
+491 21 
+96 185 96 
+29 14 13 
+5 481 11 21 470 
+70 5 70 11 
+167 42 167 217 
+72 15 73 161 172 
+82 82 
+97 297 97 
+193 182 186 183 184 177 214 
+152 152 
+163 298 7 
+39 73 71 
+490 23 23 496 488 74 23 74 486 23 23 74 
+17 17 
+170 170 483 444 443 234 
+25 472 
+5 5 11 70 69 
+149 149 455 
+356 68 477 468 17 479 66 
+159 172 6 71 6 6 158 13 494 169 
+155 44 438 144 500 
+156 9 9 
+146 146 
+173 10 10 461 
+7 6 6 
+269 48 268 
+50 100 
+323 174 18 
+69 69 22 98 
+38 171 
+22 29 489 10 
+0 0 
+11 5 
+29 13 14 232 231 451 289 452 229 
+260 11 156 
+166 160 166 39 
+223 134 134 420 
+66 401 68 132 17 84 287 5 
+39 304 
+65 84 132 
+400 211 
+145 144 
+16 28 254 48 50 100 42 154 262 133 17 
+0 0 
+28 28 
+11 476 464 
+61 61 86 86 
+38 38 
+463 478 
+437 265 
+22 39 485 171 98 
+434 51 344 
+16 16 
+67 67 67 448 
+22 12 161 
+15 377 147 147 374 
+119 317 0 
+38 484 
+403 499 
+432 442 
+28 0 16 50 465 42 
+163 487 7 162 
+99 99 325 423 83 83 
+154 133 
+5 37 492 235 160 279 
+10 10 457 493 10 460 
+441 4 4 4 4 4 4 4 
+153 153 
+159 164 164 
+328 37 
+65 65 404 347 431 459 
+80 80 44 44 
+61 446 
+162 495 7 453 
+157 21 204 68 37 66 469 145 
+37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 
+430 445 433 
+449 14 
+9 9 9 9 
+440 238 226 
+148 148 
+266 267 181 
+48 498 
+263 255 256 
+458 158 7 
+72 168 12 165 71 73 173 49 
+0 0 
+7 7 6 
+14 29 13 6 15 14 15 13 
+480 439 21 
+450 21 151 
+12 12 49 14 13 165 12 169 72 15 15 
+91 91 
+22 12 49 168 
+497 101 30 411 30 482 30 53 30 101 176 415 53 447 
+462 150 150 
+471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 

models/multitask/share-bottom/data/train/train_data

+0,0,73,0,0,0,0,1700.09,0,0,2,0,95,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,58,0,0,0,0,1053.55,1,0,2,52,94,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+1,0,18,0,0,0,0,991.95,0,0,2,0,95,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0
+1,0,9,0,0,0,0,1758.14,0,0,0,0,94,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+1,0,10,0,0,0,0,1069.16,0,0,0,0,94,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,48,1200,0,0,0,162.61,1,2,2,52,95,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,42,0,5178,0,0,1535.86,6,0,2,52,94,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+1,0,28,0,0,0,0,898.83,4,0,2,30,95,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,47,876,0,0,0,1661.53,5,0,2,52,95,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0
+0,0,34,0,0,0,0,1146.79,6,0,2,52,94,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0

models/multitask/share-bottom/model.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import paddle.fluid as fluid
+
+from fleetrec.core.utils import envs
+from fleetrec.core.model import Model as ModelBase
+
+
+class Model(ModelBase):
+    def __init__(self, config):
+        ModelBase.__init__(self, config)
+
+    def train(self):
+
+        feature_size = envs.get_global_env("hyper_parameters.feature_size", None, self._namespace)
+        bottom_size = envs.get_global_env("hyper_parameters.bottom_size", None, self._namespace)
+        tower_size = envs.get_global_env("hyper_parameters.tower_size", None, self._namespace)
+        tower_nums = envs.get_global_env("hyper_parameters.tower_nums", None, self._namespace)
+
+        input_data = fluid.data(name="input", shape=[-1, feature_size], dtype="float32")
+        label_income = fluid.data(name="label_income", shape=[-1, 2], dtype="float32", lod_level=0)
+        label_marital = fluid.data(name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0)
+        
+        self._data_var.extend([input_data, label_income, label_marital])
+
+        bottom_output = fluid.layers.fc(input=input_data,
+                                           size=bottom_size,
+                                           act='relu',
+                                           bias_attr=fluid.ParamAttr(learning_rate=1.0),
+                                           name='bottom_output')
+      
+       
+        # Build tower layer from bottom layer
+        output_layers = []
+        for index in range(tower_nums):    
+            tower_layer = fluid.layers.fc(input=bottom_output,
+                                       size=tower_size,
+                                       act='relu',
+                                       name='task_layer_' + str(index))
+            output_layer = fluid.layers.fc(input=tower_layer,
+                                       size=2,
+                                       act='softmax',
+                                       name='output_layer_' + str(index))
+            output_layers.append(output_layer)
+
+
+        pred_income = fluid.layers.clip(output_layers[0], min=1e-15, max=1.0 - 1e-15)
+        pred_marital = fluid.layers.clip(output_layers[1], min=1e-15, max=1.0 - 1e-15)
+
+        cost_income = fluid.layers.cross_entropy(input=pred_income, label=label_income,soft_label = True)
+        cost_marital = fluid.layers.cross_entropy(input=pred_marital, label=label_marital,soft_label = True)
+        
+
+        label_income_1 = fluid.layers.slice(label_income, axes=[1], starts=[1], ends=[2])
+        label_marital_1 = fluid.layers.slice(label_marital, axes=[1], starts=[1], ends=[2])
+        
+        auc_income, batch_auc_1, auc_states_1  = fluid.layers.auc(input=pred_income, label=fluid.layers.cast(x=label_income_1, dtype='int64'))
+        auc_marital, batch_auc_2, auc_states_2 = fluid.layers.auc(input=pred_marital, label=fluid.layers.cast(x=label_marital_1, dtype='int64'))
+
+        cost = fluid.layers.elementwise_add(cost_income, cost_marital, axis=1)
+        
+        avg_cost =  fluid.layers.mean(x=cost)
+    
+        self._cost = avg_cost
+        self._metrics["AUC_income"] = auc_income
+        self._metrics["BATCH_AUC_income"] = batch_auc_1
+        self._metrics["AUC_marital"] = auc_marital
+        self._metrics["BATCH_AUC_marital"] = batch_auc_2
+
+
+    def train_net(self):
+        self.train()
+
+
+    def infer_net(self):
+        pass

models/recall/gru4rec/__init__.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

models/recall/gru4rec/config.yaml

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+train:
+  trainer:
+    # for cluster training
+    strategy: "async"
+
+  epochs: 3
+  workspace: "fleetrec.models.recall.gru4rec"
+
+  reader:
+    batch_size: 5
+    class: "{workspace}/rsc15_reader.py"
+    train_data_path: "{workspace}/data/train"
+
+  model:
+    models: "{workspace}/model.py"
+    hyper_parameters:
+      vocab_size: 1000
+      hid_size: 100
+      emb_lr_x: 10.0
+      gru_lr_x: 1.0
+      fc_lr_x: 1.0
+      init_low_bound: -0.04
+      init_high_bound: 0.04
+      learning_rate: 0.01
+      optimizer: adagrad
+
+  save:
+    increment:
+      dirname: "increment"
+      epoch_interval: 2
+      save_last: True
+    inference:
+      dirname: "inference"
+      epoch_interval: 4
+      save_last: True

models/recall/gru4rec/data/test/small_test.txt

+0 16 
+475 473 155 
+491 21 
+96 185 96 
+29 14 13 
+5 481 11 21 470 
+70 5 70 11 
+167 42 167 217 
+72 15 73 161 172 
+82 82 
+97 297 97 
+193 182 186 183 184 177 214 
+152 152 
+163 298 7 
+39 73 71 
+490 23 23 496 488 74 23 74 486 23 23 74 
+17 17 
+170 170 483 444 443 234 
+25 472 
+5 5 11 70 69 
+149 149 455 
+356 68 477 468 17 479 66 
+159 172 6 71 6 6 158 13 494 169 
+155 44 438 144 500 
+156 9 9 
+146 146 
+173 10 10 461 
+7 6 6 
+269 48 268 
+50 100 
+323 174 18 
+69 69 22 98 
+38 171 
+22 29 489 10 
+0 0 
+11 5 
+29 13 14 232 231 451 289 452 229 
+260 11 156 
+166 160 166 39 
+223 134 134 420 
+66 401 68 132 17 84 287 5 
+39 304 
+65 84 132 
+400 211 
+145 144 
+16 28 254 48 50 100 42 154 262 133 17 
+0 0 
+28 28 
+11 476 464 
+61 61 86 86 
+38 38 
+463 478 
+437 265 
+22 39 485 171 98 
+434 51 344 
+16 16 
+67 67 67 448 
+22 12 161 
+15 377 147 147 374 
+119 317 0 
+38 484 
+403 499 
+432 442 
+28 0 16 50 465 42 
+163 487 7 162 
+99 99 325 423 83 83 
+154 133 
+5 37 492 235 160 279 
+10 10 457 493 10 460 
+441 4 4 4 4 4 4 4 
+153 153 
+159 164 164 
+328 37 
+65 65 404 347 431 459 
+80 80 44 44 
+61 446 
+162 495 7 453 
+157 21 204 68 37 66 469 145 
+37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 
+430 445 433 
+449 14 
+9 9 9 9 
+440 238 226 
+148 148 
+266 267 181 
+48 498 
+263 255 256 
+458 158 7 
+72 168 12 165 71 73 173 49 
+0 0 
+7 7 6 
+14 29 13 6 15 14 15 13 
+480 439 21 
+450 21 151 
+12 12 49 14 13 165 12 169 72 15 15 
+91 91 
+22 12 49 168 
+497 101 30 411 30 482 30 53 30 101 176 415 53 447 
+462 150 150 
+471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 

models/recall/gru4rec/data/train/small_train.txt

+197 196 198 236 
+93 93 384 362 363 43 
+336 364 407 
+421 322 
+314 388 
+128 58 
+138 138 
+46 46 46 
+34 34 57 57 57 342 228 321 346 357 59 376 
+110 110 
+135 94 135 
+27 250 27 
+129 118 
+18 18 18 
+81 81 89 89 
+27 27 
+20 20 20 20 20 212 
+33 33 33 33 
+62 62 62 63 63 55 248 124 381 428 383 382 43 43 261 63 
+90 90 78 78 
+399 397 202 141 104 104 245 192 191 271 
+239 332 283 88 
+187 313 
+136 136 324 
+41 41 
+352 128 
+413 414 
+410 45 45 45 1 1 1 1 1 1 1 1 31 31 31 31 
+92 334 92 
+95 285 
+215 249 
+390 41 
+116 116 
+300 252 
+2 2 2 2 2 
+8 8 8 8 8 8 
+53 241 259 
+118 129 126 94 137 208 216 299 
+209 368 139 418 419 
+311 180 
+303 302 203 284 
+369 32 32 32 32 337 
+207 47 47 47 
+106 107 
+143 143 
+179 178 
+109 109 
+405 79 79 371 246 
+251 417 427 
+333 88 387 358 123 348 394 360 36 365 
+3 3 3 3 3 
+189 188 
+398 425 
+107 406 
+281 201 141 
+2 2 2 
+359 54 
+395 385 293 
+60 60 60 121 121 233 58 58 
+24 199 175 24 24 24 351 386 106 
+115 294 
+122 122 127 127 
+35 35 
+282 393 
+277 140 140 343 225 123 36 36 36 221 114 114 59 59 117 117 247 367 219 258 222 301 375 350 353 111 111 
+275 272 273 274 331 330 305 108 76 76 108 
+26 26 26 408 26 
+290 18 210 291 
+372 139 424 113 
+341 340 335 
+120 370 
+224 200 
+426 416 
+137 319 
+402 55 
+54 54 
+327 119 
+125 125 
+391 396 354 355 389 
+142 142 
+295 320 
+113 366 
+253 85 85 
+56 56 310 309 308 307 278 25 25 19 19 3 312 19 19 19 3 25 
+220 338 
+34 130 
+130 120 380 315 
+339 422 
+379 378 
+95 56 392 115 
+55 124 
+126 34 
+349 373 361 
+195 194 
+75 75 
+64 64 64 
+35 35 
+40 40 40 242 77 244 77 243 
+257 316 
+103 306 102 51 52 103 105 52 52 292 318 112 286 345 237 276 112 51 102 105 

models/recall/gru4rec/model.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import paddle.fluid as fluid
+
+from fleetrec.core.utils import envs
+from fleetrec.core.model import Model as ModelBase
+
+
+class Model(ModelBase):
+    def __init__(self, config):
+        ModelBase.__init__(self, config)
+
+    def all_vocab_network(self):
+        """ network definition """
+        recall_k = envs.get_global_env("hyper_parameters.recall_k", None, self._namespace)
+        vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace)
+        hid_size = envs.get_global_env("hyper_parameters.hid_size", None, self._namespace)
+        init_low_bound = envs.get_global_env("hyper_parameters.init_low_bound", None, self._namespace)
+        init_high_bound = envs.get_global_env("hyper_parameters.init_high_bound", None, self._namespace)
+        emb_lr_x = envs.get_global_env("hyper_parameters.emb_lr_x", None, self._namespace)
+        gru_lr_x = envs.get_global_env("hyper_parameters.gru_lr_x", None, self._namespace)
+        fc_lr_x = envs.get_global_env("hyper_parameters.fc_lr_x", None, self._namespace)
+        # Input data
+        src_wordseq = fluid.data(
+            name="src_wordseq", shape=[None, 1], dtype="int64", lod_level=1)
+        dst_wordseq = fluid.data(
+            name="dst_wordseq", shape=[None, 1], dtype="int64", lod_level=1)
+
+        emb = fluid.embedding(
+            input=src_wordseq,
+            size=[vocab_size, hid_size],
+            param_attr=fluid.ParamAttr(
+                initializer=fluid.initializer.Uniform(
+                    low=init_low_bound, high=init_high_bound),
+                learning_rate=emb_lr_x),
+            is_sparse=True)
+        fc0 = fluid.layers.fc(input=emb,
+                              size=hid_size * 3,
+                              param_attr=fluid.ParamAttr(
+                                  initializer=fluid.initializer.Uniform(
+                                      low=init_low_bound, high=init_high_bound),
+                                  learning_rate=gru_lr_x))
+        gru_h0 = fluid.layers.dynamic_gru(
+            input=fc0,
+            size=hid_size,
+            param_attr=fluid.ParamAttr(
+                initializer=fluid.initializer.Uniform(
+                    low=init_low_bound, high=init_high_bound),
+                learning_rate=gru_lr_x))
+
+        fc = fluid.layers.fc(input=gru_h0,
+                             size=vocab_size,
+                             act='softmax',
+                             param_attr=fluid.ParamAttr(
+                                 initializer=fluid.initializer.Uniform(
+                                     low=init_low_bound, high=init_high_bound),
+                                 learning_rate=fc_lr_x))
+        cost = fluid.layers.cross_entropy(input=fc, label=dst_wordseq)
+        acc = fluid.layers.accuracy(input=fc, label=dst_wordseq, k=recall_k)
+        avg_cost = fluid.layers.mean(x=cost)
+
+        self._data_var.append(src_wordseq)
+        self._data_var.append(dst_wordseq)
+        self._cost = avg_cost
+        self._metrics["cost"] = avg_cost
+        self._metrics["acc"] = acc
+
+
+    def train_net(self):
+        self.all_vocab_network()
+
+
+    def infer_net(self):
+        pass

models/recall/gru4rec/rsc15_reader.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+from fleetrec.core.reader import Reader
+from fleetrec.core.utils import envs
+
+
+class TrainReader(Reader):
+    def init(self):
+        pass
+
+    def generate_sample(self, line):
+        """
+        Read the data line by line and process it as a dictionary
+        """
+
+        def reader():
+            """
+            This function needs to be implemented by the user, based on data format
+            """
+            l = line.strip().split()
+            l = [w for w in l]
+            src_seq = l[:len(l) - 1]
+            src_seq = [int(e) for e in src_seq]
+            trg_seq = l[1:]
+            trg_seq = [int(e) for e in trg_seq]
+            feature_name = ["src_wordseq", "dst_wordseq"]
+            yield zip(feature_name, [src_seq] + [trg_seq])
+
+        return reader

models/recall/ssr/__init__.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

models/recall/ssr/config.yaml

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+train:
+  trainer:
+    # for cluster training
+    strategy: "async"
+
+  epochs: 3
+  workspace: "fleetrec.models.recall.ssr"
+
+  reader:
+    batch_size: 5
+    class: "{workspace}/ssr_reader.py"
+    train_data_path: "{workspace}/data/train"
+
+  model:
+    models: "{workspace}/model.py"
+    hyper_parameters:
+      vocab_size: 1000
+      emb_dim: 128
+      hidden_size: 100
+      learning_rate: 0.01
+      optimizer: adagrad
+
+  save:
+    increment:
+      dirname: "increment"
+      epoch_interval: 2
+      save_last: True
+    inference:
+      dirname: "inference"
+      epoch_interval: 4
+      save_last: True

models/recall/ssr/data/test/small_test.txt

+0 16 
+475 473 155 
+491 21 
+96 185 96 
+29 14 13 
+5 481 11 21 470 
+70 5 70 11 
+167 42 167 217 
+72 15 73 161 172 
+82 82 
+97 297 97 
+193 182 186 183 184 177 214 
+152 152 
+163 298 7 
+39 73 71 
+490 23 23 496 488 74 23 74 486 23 23 74 
+17 17 
+170 170 483 444 443 234 
+25 472 
+5 5 11 70 69 
+149 149 455 
+356 68 477 468 17 479 66 
+159 172 6 71 6 6 158 13 494 169 
+155 44 438 144 500 
+156 9 9 
+146 146 
+173 10 10 461 
+7 6 6 
+269 48 268 
+50 100 
+323 174 18 
+69 69 22 98 
+38 171 
+22 29 489 10 
+0 0 
+11 5 
+29 13 14 232 231 451 289 452 229 
+260 11 156 
+166 160 166 39 
+223 134 134 420 
+66 401 68 132 17 84 287 5 
+39 304 
+65 84 132 
+400 211 
+145 144 
+16 28 254 48 50 100 42 154 262 133 17 
+0 0 
+28 28 
+11 476 464 
+61 61 86 86 
+38 38 
+463 478 
+437 265 
+22 39 485 171 98 
+434 51 344 
+16 16 
+67 67 67 448 
+22 12 161 
+15 377 147 147 374 
+119 317 0 
+38 484 
+403 499 
+432 442 
+28 0 16 50 465 42 
+163 487 7 162 
+99 99 325 423 83 83 
+154 133 
+5 37 492 235 160 279 
+10 10 457 493 10 460 
+441 4 4 4 4 4 4 4 
+153 153 
+159 164 164 
+328 37 
+65 65 404 347 431 459 
+80 80 44 44 
+61 446 
+162 495 7 453 
+157 21 204 68 37 66 469 145 
+37 151 230 206 240 205 264 87 409 87 288 270 280 329 157 296 454 474 
+430 445 433 
+449 14 
+9 9 9 9 
+440 238 226 
+148 148 
+266 267 181 
+48 498 
+263 255 256 
+458 158 7 
+72 168 12 165 71 73 173 49 
+0 0 
+7 7 6 
+14 29 13 6 15 14 15 13 
+480 439 21 
+450 21 151 
+12 12 49 14 13 165 12 169 72 15 15 
+91 91 
+22 12 49 168 
+497 101 30 411 30 482 30 53 30 101 176 415 53 447 
+462 150 150 
+471 456 131 435 131 467 436 412 227 218 190 466 429 213 326 

models/recall/ssr/data/train/small_train.txt

+197 196 198 236 
+93 93 384 362 363 43 
+336 364 407 
+421 322 
+314 388 
+128 58 
+138 138 
+46 46 46 
+34 34 57 57 57 342 228 321 346 357 59 376 
+110 110 
+135 94 135 
+27 250 27 
+129 118 
+18 18 18 
+81 81 89 89 
+27 27 
+20 20 20 20 20 212 
+33 33 33 33 
+62 62 62 63 63 55 248 124 381 428 383 382 43 43 261 63 
+90 90 78 78 
+399 397 202 141 104 104 245 192 191 271 
+239 332 283 88 
+187 313 
+136 136 324 
+41 41 
+352 128 
+413 414 
+410 45 45 45 1 1 1 1 1 1 1 1 31 31 31 31 
+92 334 92 
+95 285 
+215 249 
+390 41 
+116 116 
+300 252 
+2 2 2 2 2 
+8 8 8 8 8 8 
+53 241 259 
+118 129 126 94 137 208 216 299 
+209 368 139 418 419 
+311 180 
+303 302 203 284 
+369 32 32 32 32 337 
+207 47 47 47 
+106 107 
+143 143 
+179 178 
+109 109 
+405 79 79 371 246 
+251 417 427 
+333 88 387 358 123 348 394 360 36 365 
+3 3 3 3 3 
+189 188 
+398 425 
+107 406 
+281 201 141 
+2 2 2 
+359 54 
+395 385 293 
+60 60 60 121 121 233 58 58 
+24 199 175 24 24 24 351 386 106 
+115 294 
+122 122 127 127 
+35 35 
+282 393 
+277 140 140 343 225 123 36 36 36 221 114 114 59 59 117 117 247 367 219 258 222 301 375 350 353 111 111 
+275 272 273 274 331 330 305 108 76 76 108 
+26 26 26 408 26 
+290 18 210 291 
+372 139 424 113 
+341 340 335 
+120 370 
+224 200 
+426 416 
+137 319 
+402 55 
+54 54 
+327 119 
+125 125 
+391 396 354 355 389 
+142 142 
+295 320 
+113 366 
+253 85 85 
+56 56 310 309 308 307 278 25 25 19 19 3 312 19 19 19 3 25 
+220 338 
+34 130 
+130 120 380 315 
+339 422 
+379 378 
+95 56 392 115 
+55 124 
+126 34 
+349 373 361 
+195 194 
+75 75 
+64 64 64 
+35 35 
+40 40 40 242 77 244 77 243 
+257 316 
+103 306 102 51 52 103 105 52 52 292 318 112 286 345 237 276 112 51 102 105 

models/recall/ssr/model.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import paddle.fluid as fluid
+
+from fleetrec.core.utils import envs
+from fleetrec.core.model import Model as ModelBase
+import paddle.fluid.layers.tensor as tensor
+import paddle.fluid.layers.io as io
+import paddle.fluid.layers.control_flow as cf
+
+
+
+class BowEncoder(object):
+    """ bow-encoder """
+
+    def __init__(self):
+        self.param_name = ""
+
+    def forward(self, emb):
+        return fluid.layers.sequence_pool(input=emb, pool_type='sum')
+
+
+class GrnnEncoder(object):
+    """ grnn-encoder """
+
+    def __init__(self, param_name="grnn", hidden_size=128):
+        self.param_name = param_name
+        self.hidden_size = hidden_size
+
+    def forward(self, emb):
+        fc0 = fluid.layers.fc(input=emb,
+                              size=self.hidden_size * 3,
+                              param_attr=self.param_name + "_fc.w",
+                              bias_attr=False)
+
+        gru_h = fluid.layers.dynamic_gru(
+            input=fc0,
+            size=self.hidden_size,
+            is_reverse=False,
+            param_attr=self.param_name + ".param",
+            bias_attr=self.param_name + ".bias")
+        return fluid.layers.sequence_pool(input=gru_h, pool_type='max')
+
+class PairwiseHingeLoss(object):
+    def __init__(self, margin=0.8):
+        self.margin = margin
+
+    def forward(self, pos, neg):
+        loss_part1 = fluid.layers.elementwise_sub(
+            tensor.fill_constant_batch_size_like(
+                input=pos, shape=[-1, 1], value=self.margin, dtype='float32'),
+            pos)
+        loss_part2 = fluid.layers.elementwise_add(loss_part1, neg)
+        loss_part3 = fluid.layers.elementwise_max(
+            tensor.fill_constant_batch_size_like(
+                input=loss_part2, shape=[-1, 1], value=0.0, dtype='float32'),
+            loss_part2)
+        return loss_part3
+
+class Model(ModelBase):
+    def __init__(self, config):
+        ModelBase.__init__(self, config)
+
+    def get_correct(self, x, y):
+        less = tensor.cast(cf.less_than(x, y), dtype='float32')
+        correct = fluid.layers.reduce_sum(less)
+        return correct
+
+    def train(self):
+        
+        vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace)
+        emb_dim = envs.get_global_env("hyper_parameters.emb_dim", None, self._namespace)
+        hidden_size = envs.get_global_env("hyper_parameters.hidden_size", None, self._namespace)
+        emb_shape = [vocab_size, emb_dim]
+
+        self.user_encoder = GrnnEncoder()
+        self.item_encoder = BowEncoder()
+        self.pairwise_hinge_loss = PairwiseHingeLoss()
+
+        user_data = fluid.data(
+            name="user", shape=[None, 1], dtype="int64", lod_level=1)
+        pos_item_data = fluid.data(
+            name="p_item", shape=[None, 1], dtype="int64", lod_level=1)
+        neg_item_data = fluid.data(
+            name="n_item", shape=[None, 1], dtype="int64", lod_level=1)
+        self._data_var.extend([user_data, pos_item_data, neg_item_data])
+
+        user_emb = fluid.embedding(
+            input=user_data, size=emb_shape, param_attr="emb.item")
+        pos_item_emb = fluid.embedding(
+            input=pos_item_data, size=emb_shape, param_attr="emb.item")
+        neg_item_emb = fluid.embedding(
+            input=neg_item_data, size=emb_shape, param_attr="emb.item")
+        user_enc = self.user_encoder.forward(user_emb)
+        pos_item_enc = self.item_encoder.forward(pos_item_emb)
+        neg_item_enc = self.item_encoder.forward(neg_item_emb)
+        user_hid = fluid.layers.fc(input=user_enc,
+                                   size=hidden_size,
+                                   param_attr='user.w',
+                                   bias_attr="user.b")
+        pos_item_hid = fluid.layers.fc(input=pos_item_enc,
+                                       size=hidden_size,
+                                       param_attr='item.w',
+                                       bias_attr="item.b")
+        neg_item_hid = fluid.layers.fc(input=neg_item_enc,
+                                       size=hidden_size,
+                                       param_attr='item.w',
+                                       bias_attr="item.b")
+        cos_pos = fluid.layers.cos_sim(user_hid, pos_item_hid)
+        cos_neg = fluid.layers.cos_sim(user_hid, neg_item_hid)
+        hinge_loss = self.pairwise_hinge_loss.forward(cos_pos, cos_neg)
+        avg_cost = fluid.layers.mean(hinge_loss)
+        correct = self.get_correct(cos_neg, cos_pos)
+        
+        self._cost = avg_cost
+        self._metrics["correct"] = correct
+        self._metrics["hinge_loss"] = hinge_loss
+
+
+    def train_net(self):
+        self.train()
+
+
+    def infer_net(self):
+        pass

models/recall/ssr/ssr_reader.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+from fleetrec.core.reader import Reader
+from fleetrec.core.utils import envs
+import random
+
+
+class TrainReader(Reader):
+    def init(self):
+        pass
+
+    def sample_neg_from_seq(self, seq):
+        return seq[random.randint(0, len(seq) - 1)]
+
+
+    def generate_sample(self, line):
+        """
+        Read the data line by line and process it as a dictionary
+        """
+
+        def reader():
+            """
+            This function needs to be implemented by the user, based on data format
+            """
+            ids = line.strip().split()
+            conv_ids = [int(i) for i in ids]
+            boundary = len(ids) - 1
+            src = conv_ids[:boundary]
+            pos_tgt = [conv_ids[boundary]]
+            neg_tgt = [self.sample_neg_from_seq(src)]
+            feature_name = ["user", "p_item", "n_item"]
+            yield zip(feature_name, [src] + [pos_tgt] + [neg_tgt])
+
+        return reader