mpi训练时cost恒为0
Created by: cszhou
本地训练网络的cost会有变化。但是mpi训练，第一个batch就发现cost为0，接下来cost一直保持不变。请教是什么原因。
目前打印过输入的训练数据，数据看起来没有异常。
I1115 15:30:28.423450 2915 PyDataProvider2.cpp:611] Scanner created.
I1115 15:30:28.468631 2915 PyDataProvider2.cpp:643] Reading CPU Batch Done.
I1115 15:30:44.074748 2915 TrainerInternal.cpp:165] Batch=1 samples=512 AvgCost=0 CurrentCost=0 Eval: __auc_evaluator_0__=0 CurrentEval: __auc_evaluator_0__=0
I1115 15:30:44.179998 2915 Stat.cpp:130] ======= StatSet: [GlobalStatInfo] status ======
I1115 15:30:44.180039 2915 Stat.cpp:133] Stat=sendSparseParam TID=3055 total=12631 avg=12631 max=12631 min=12631 count=1
I1115 15:30:44.180058 2915 Stat.cpp:133] Stat=clearGradient TID=3317 total=1596.19 avg=1596.19 max=1596.19 min=1596.19 count=1
Stat=clearGradient TID=3320 total=2108.17 avg=2108.17 max=2108.17 min=2108.17 count=1
Stat=clearGradient TID=3324 total=2166.38 avg=2166.38 max=2166.38 min=2166.38 count=1
Stat=clearGradient TID=3318 total=2249.74 avg=2249.74 max=2249.74 min=2249.74 count=1
Stat=clearGradient TID=3319 total=2274.27 avg=2274.27 max=2274.27 min=2274.27 count=1
Stat=clearGradient TID=3321 total=2275.64 avg=2275.64 max=2275.64 min=2275.64 count=1
Stat=clearGradient TID=3322 total=2275.71 avg=2275.71 max=2275.71 min=2275.71 count=1
Stat=clearGradient TID=3323 total=2302.63 avg=2302.63 max=2302.63 min=2302.63 count=1
I1115 15:30:44.180104 2915 Stat.cpp:133] Stat=PyDP2.getNextBatchInternal TID=2915 total=290085 avg=290085 max=290085 min=290085 count=1
I1115 15:30:44.180115 2915 Stat.cpp:133] Stat=getParamSparse TID=3055 total=43.291 avg=43.291 max=43.291 min=43.291 count=1
I1115 15:30:44.180127 2915 Stat.cpp:133] Stat=prefetch TID=2915 total=64.268 avg=64.268 max=64.268 min=64.268 count=1
I1115 15:30:44.180140 2915 Stat.cpp:133] Stat=wait_value TID=3317 total=0 avg=0 max=0 min=0 count=1
Stat=wait_value TID=3320 total=0.001 avg=0.001 max=0.001 min=0.001 count=1
Stat=wait_value TID=3324 total=0 avg=0 max=0 min=0 count=1
Stat=wait_value TID=3318 total=0.001 avg=0.001 max=0.001 min=0.001 count=1
Stat=wait_value TID=3319 total=0 avg=0 max=0 min=0 count=1
Stat=wait_value TID=3321 total=0.001 avg=0.001 max=0.001 min=0.001 count=1
Stat=wait_value TID=3322 total=0 avg=0 max=0 min=0 count=1
Stat=wait_value TID=3323 total=0 avg=0 max=0 min=0 count=1
I1115 15:30:44.180238 2915 Stat.cpp:133] Stat=TrainBatch TID=2915 total=15711.2 avg=15711.2 max=15711.2 min=15711.2 count=1
I1115 15:30:44.180255 2915 Stat.cpp:133] Stat=waitOutArgs TID=2915 total=2302.62 avg=287.828 max=1596.21 min=0.001 count=8
I1115 15:30:44.180269 2915 Stat.cpp:133] Stat=getTrainBatch TID=2915 total=290085 avg=290085 max=290085 min=290085 count=1
I1115 15:30:44.180279 2915 Stat.cpp:133] Stat=forwardBackward TID=2915 total=2759.93 avg=2759.93 max=2759.93 min=2759.93 count=1
I1115 15:30:44.180292 2915 Stat.cpp:133] Stat=finishBatch TID=2915 total=12631.1 avg=12631.1 max=12631.1 min=12631.1 count=1
I1115 15:30:44.180305 2915 Stat.cpp:133] Stat=client_sendAndRecv_send TID=7585 total=67.639 avg=33.819 max=36.404 min=31.235 count=2
Stat=client_sendAndRecv_send TID=7601 total=309.091 avg=154.545 max=179.962 min=129.129 count=2
Stat=client_sendAndRecv_send TID=7599 total=285.496 avg=142.748 max=151.026 min=134.47 count=2
Stat=client_sendAndRecv_send TID=7594 total=313.316 avg=156.658 max=161.559 min=151.757 count=2
Stat=client_sendAndRecv_send TID=7595 total=318.813 avg=159.406 max=159.71 min=159.103 count=2
Stat=client_sendAndRecv_send TID=7586 total=324.149 avg=162.074 max=162.552 min=161.597 count=2
Stat=client_sendAndRecv_send TID=7587 total=316.404 avg=158.202 max=162.621 min=153.783 count=2
Stat=client_sendAndRecv_send TID=7591 total=507.3 avg=253.65 max=344.283 min=163.017 count=2
Stat=client_sendAndRecv_send TID=7593 total=325.368 avg=162.684 max=164.66 min=160.708 count=2
Stat=client_sendAndRecv_send TID=7603 total=324.085 avg=162.042 max=164.879 min=159.206 count=2
Stat=client_sendAndRecv_send TID=7596 total=450.1 avg=225.05 max=283.793 min=166.307 count=2
Stat=client_sendAndRecv_send TID=7602 total=331.816 avg=165.908 max=167.097 min=164.719 count=2
Stat=client_sendAndRecv_send TID=7590 total=330.372 avg=165.186 max=167.269 min=163.103 count=2
Stat=client_sendAndRecv_send TID=7588 total=331.817 avg=165.908 max=167.757 min=164.06 count=2
Stat=client_sendAndRecv_send TID=7592 total=331.824 avg=165.912 max=167.808 min=164.016 count=2
Stat=client_sendAndRecv_send TID=7604 total=330.974 avg=165.487 max=168.348 min=162.626 count=2
Stat=client_sendAndRecv_send TID=7597 total=470.18 avg=235.09 max=284.13 min=186.05 count=2
Stat=client_sendAndRecv_send TID=7598 total=377.881 avg=188.94 max=214.039 min=163.842 count=2
Stat=client_sendAndRecv_send TID=7600 total=506.221 avg=253.11 max=291.278 min=214.943 count=2
Stat=client_sendAndRecv_send TID=7589 total=466.909 avg=233.454 max=305.997 min=160.912 count=2
Stat=client_sendAndRecv_send TID=7574 total=2546.58 avg=848.859 max=2545.53 min=0.397 count=3
Stat=client_sendAndRecv_send TID=7578 total=2666.93 avg=888.978 max=2665.68 min=0.431 count=3
Stat=client_sendAndRecv_send TID=7576 total=2697.64 avg=899.213 max=2696.71 min=0.169 count=3
Stat=client_sendAndRecv_send TID=7563 total=2706.02 avg=902.006 max=2705.02 min=0.139 count=3
Stat=client_sendAndRecv_send TID=7568 total=2936.91 avg=978.97 max=2935.77 min=0.474 count=3
Stat=client_sendAndRecv_send TID=7573 total=3083.46 avg=1027.82 max=3082.01 min=0.554 count=3
Stat=client_sendAndRecv_send TID=7564 total=3291.76 avg=1097.25 max=3290.58 min=0.307 count=3
Stat=client_sendAndRecv_send TID=7570 total=3350.56 avg=1116.85 max=3348.91 min=0.278 count=3
Stat=client_sendAndRecv_send TID=7580 total=3430.14 avg=1143.38 max=3428.91 min=0.425 count=3
Stat=client_sendAndRecv_send TID=7572 total=3482.64 avg=1160.88 max=3481.21 min=0.455 count=3
Stat=client_sendAndRecv_send TID=7582 total=3530.81 avg=1176.94 max=3530.14 min=0.296 count=3
Stat=client_sendAndRecv_send TID=7577 total=3531.34 avg=1177.11 max=3530.16 min=0.237 count=3
Stat=client_sendAndRecv_send TID=7566 total=3574.35 avg=1191.45 max=3573.46 min=0.37 count=3
Stat=client_sendAndRecv_send TID=7569 total=3585.07 avg=1195.02 max=3583.98 min=0.247 count=3
Stat=client_sendAndRecv_send TID=7575 total=3588.39 avg=1196.13 max=3587.25 min=0.444 count=3
Stat=client_sendAndRecv_send TID=7579 total=3587.49 avg=1195.83 max=3585.95 min=0.504 count=3
Stat=client_sendAndRecv_send TID=7571 total=3599.68 avg=1199.89 max=3598.16 min=0.529 count=3
Stat=client_sendAndRecv_send TID=7565 total=3608.49 avg=1202.83 max=3607.17 min=0.527 count=3
Stat=client_sendAndRecv_send TID=7567 total=3702.2 avg=1234.07 max=3700.89 min=0.584 count=3
Stat=client_sendAndRecv_send TID=7581 total=3962.36 avg=1320.79 max=3961.32 min=0.336 count=3
I1115 15:30:44.180513 2915 Stat.cpp:133] Stat=client_sendAndRecv_recv TID=7585 total=11996 avg=5997.99 max=11913.9 min=82.038 count=2
Stat=client_sendAndRecv_recv TID=7601 total=11818.5 avg=5909.26 max=11597.7 min=220.791 count=2
Stat=client_sendAndRecv_recv TID=7599 total=12263.1 avg=6131.56 max=12038.9 min=224.22 count=2
Stat=client_sendAndRecv_recv TID=7594 total=11692.8 avg=5846.42 max=11484.4 min=208.413 count=2
Stat=client_sendAndRecv_recv TID=7597 total=12123.5 avg=6061.74 max=11947.3 min=176.218 count=2
Stat=client_sendAndRecv_recv TID=7591 total=12033.3 avg=6016.64 max=11827.5 min=205.81 count=2
Stat=client_sendAndRecv_recv TID=7592 total=11738.3 avg=5869.15 max=11536.7 min=201.602 count=2
Stat=client_sendAndRecv_recv TID=7587 total=12260.7 avg=6130.36 max=12051.2 min=209.478 count=2
Stat=client_sendAndRecv_recv TID=7603 total=12128.9 avg=6064.47 max=11921.7 min=207.265 count=2
Stat=client_sendAndRecv_recv TID=7595 total=11881.4 avg=5940.72 max=11666.8 min=214.593 count=2
Stat=client_sendAndRecv_recv TID=7586 total=12319.9 avg=6159.95 max=12106.5 min=213.362 count=2
Stat=client_sendAndRecv_recv TID=7593 total=12187.6 avg=6093.82 max=11975.7 min=211.971 count=2
Stat=client_sendAndRecv_recv TID=7596 total=11851.8 avg=5925.88 max=11641 min=210.795 count=2
Stat=client_sendAndRecv_recv TID=7604 total=12302.4 avg=6151.22 max=12091.2 min=211.289 count=2
Stat=client_sendAndRecv_recv TID=7590 total=12231.3 avg=6115.65 max=12018.4 min=212.955 count=2
Stat=client_sendAndRecv_recv TID=7588 total=11727.3 avg=5863.64 max=11513.6 min=213.711 count=2
Stat=client_sendAndRecv_recv TID=7598 total=11694.9 avg=5847.47 max=11526.5 min=168.474 count=2
Stat=client_sendAndRecv_recv TID=7600 total=11113.1 avg=5556.53 max=10944.3 min=168.743 count=2
Stat=client_sendAndRecv_recv TID=7602 total=11899.5 avg=5949.77 max=11680.3 min=219.22 count=2
Stat=client_sendAndRecv_recv TID=7589 total=11802.3 avg=5901.15 max=11712.1 min=90.198 count=2
Stat=client_sendAndRecv_recv TID=7574 total=22538.9 avg=7512.96 max=11683.7 min=33.286 count=3
Stat=client_sendAndRecv_recv TID=7578 total=22555.6 avg=7518.53 max=11733.4 min=32.712 count=3
Stat=client_sendAndRecv_recv TID=7576 total=22933.2 avg=7644.4 max=11750 min=33.023 count=3
Stat=client_sendAndRecv_recv TID=7563 total=22975.5 avg=7658.51 max=11712.8 min=33.845 count=3
Stat=client_sendAndRecv_recv TID=7568 total=22831.3 avg=7610.43 max=11697.4 min=33.393 count=3
Stat=client_sendAndRecv_recv TID=7573 total=22844.6 avg=7614.85 max=11761.6 min=33.88 count=3
Stat=client_sendAndRecv_recv TID=7564 total=22854.2 avg=7618.07 max=11683.3 min=34.179 count=3
Stat=client_sendAndRecv_recv TID=7570 total=22966.5 avg=7655.52 max=11810.9 min=34.095 count=3
Stat=client_sendAndRecv_recv TID=7569 total=22662.7 avg=7554.23 max=11695.4 min=33.218 count=3
Stat=client_sendAndRecv_recv TID=7579 total=22691.3 avg=7563.75 max=11711.9 min=33.803 count=3
Stat=client_sendAndRecv_recv TID=7566 total=22762.1 avg=7587.36 max=11758.3 min=33.807 count=3
Stat=client_sendAndRecv_recv TID=7580 total=22926.2 avg=7642.08 max=11739.1 min=34.922 count=3
Stat=client_sendAndRecv_recv TID=7577 total=22837.7 avg=7612.57 max=11732.2 min=34.615 count=3
Stat=client_sendAndRecv_recv TID=7571 total=22755.9 avg=7585.28 max=11694.8 min=31.967 count=3
Stat=client_sendAndRecv_recv TID=7582 total=22840.8 avg=7613.6 max=11706 min=37.472 count=3
Stat=client_sendAndRecv_recv TID=7567 total=22991.1 avg=7663.72 max=11911.1 min=33.903 count=3
Stat=client_sendAndRecv_recv TID=7565 total=23024.7 avg=7674.9 max=11675.4 min=33.856 count=3
Stat=client_sendAndRecv_recv TID=7575 total=23982.2 avg=7994.05 max=12599.1 min=35.064 count=3
Stat=client_sendAndRecv_recv TID=7581 total=22787.3 avg=7595.76 max=11744.6 min=34.744 count=3
Stat=client_sendAndRecv_recv TID=7572 total=23539.3 avg=7846.42 max=11848.6 min=33.681 count=3
I1115 15:30:44.180706 2915 Stat.cpp:133] Stat=waitbeforeMerge TID=3317 total=823.051 avg=411.525 max=706.432 min=116.619 count=2
Stat=waitbeforeMerge TID=3320 total=233.846 avg=116.923 max=194.483 min=39.363 count=2
Stat=waitbeforeMerge TID=3324 total=160.284 avg=80.142 max=136.256 min=24.028 count=2
Stat=waitbeforeMerge TID=3318 total=52.931 avg=26.465 max=52.91 min=0.021 count=2
Stat=waitbeforeMerge TID=3321 total=28.257 avg=14.128 max=27.011 min=1.246 count=2
Stat=waitbeforeMerge TID=3322 total=27.629 avg=13.814 max=26.952 min=0.677 count=2
Stat=waitbeforeMerge TID=3319 total=82.84 avg=41.42 max=54.453 min=28.387 count=2
Stat=waitbeforeMerge TID=3323 total=5.656 avg=2.828 max=5.634 min=0.022 count=2
I1115 15:30:44.180765 2915 Stat.cpp:133] Stat=copyOutArgs TID=2915 total=0.001 avg=0.001 max=0.001 min=0.001 count=1
I1115 15:30:44.180779 2915 Stat.cpp:133] Stat=taskSem_wait TID=3317 total=384087 avg=192043 max=384025 min=61.968 count=2
Stat=taskSem_wait TID=3318 total=384087 avg=192043 max=384025 min=61.972 count=2
Stat=taskSem_wait TID=3319 total=384087 avg=192043 max=384025 min=61.982 count=2
Stat=taskSem_wait TID=3320 total=384087 avg=192043 max=384025 min=61.98 count=2
Stat=taskSem_wait TID=3321 total=384087 avg=192043 max=384025 min=61.985 count=2
Stat=taskSem_wait TID=3322 total=384087 avg=192043 max=384025 min=61.992 count=2
Stat=taskSem_wait TID=3324 total=384087 avg=192043 max=384025 min=62 count=2
Stat=taskSem_wait TID=3323 total=384087 avg=192043 max=384025 min=61.993 count=2
I1115 15:30:44.180820 2915 Stat.cpp:133] Stat=thread_forward TID=3317 total=0 avg=0 max=0 min=0 count=1
Stat=thread_forward TID=3320 total=0 avg=0 max=0 min=0 count=1
Stat=thread_forward TID=3324 total=0 avg=0 max=0 min=0 count=1
Stat=thread_forward TID=3318 total=0 avg=0 max=0 min=0 count=1
Stat=thread_forward TID=3319 total=0 avg=0 max=0 min=0 count=1
Stat=thread_forward TID=3321 total=0.001 avg=0.001 max=0.001 min=0.001 count=1
Stat=thread_forward TID=3322 total=0 avg=0 max=0 min=0 count=1
Stat=thread_forward TID=3323 total=0 avg=0 max=0 min=0 count=1
I1115 15:30:44.180851 2915 Stat.cpp:133] Stat=thread_backward TID=3317 total=1163.64 avg=1163.64 max=1163.64 min=1163.64 count=1
Stat=thread_backward TID=3320 total=651.695 avg=651.695 max=651.695 min=651.695 count=1
Stat=thread_backward TID=3324 total=593.467 avg=593.467 max=593.467 min=593.467 count=1
Stat=thread_backward TID=3322 total=484.157 avg=484.157 max=484.157 min=484.157 count=1
Stat=thread_backward TID=3321 total=484.222 avg=484.222 max=484.222 min=484.222 count=1
Stat=thread_backward TID=3318 total=510.124 avg=510.124 max=510.124 min=510.124 count=1
Stat=thread_backward TID=3323 total=457.241 avg=457.241 max=457.241 min=457.241 count=1
Stat=thread_backward TID=3319 total=485.59 avg=485.59 max=485.59 min=485.59 count=1
I1115 15:30:44.180907 2915 Stat.cpp:133] Stat=mergeGradDense TID=3317 total=339.396 avg=2.495 max=134.603 min=0.001 count=136
Stat=mergeGradDense TID=3321 total=454.859 avg=3.344 max=248.478 min=0.001 count=136
Stat=mergeGradDense TID=3323 total=450.469 avg=3.312 max=249.04 min=0.001 count=136
Stat=mergeGradDense TID=3322 total=455.439 avg=3.348 max=250.046 min=0 count=136
Stat=mergeGradDense TID=3318 total=456.098 avg=3.353 max=251.478 min=0 count=136
Stat=mergeGradDense TID=3319 total=401.6 avg=2.952 max=211.521 min=0 count=136
Stat=mergeGradDense TID=3320 total=416.759 avg=3.064 max=231.794 min=0.001 count=136
Stat=mergeGradDense TID=3324 total=432.113 avg=3.177 max=239.906 min=0.001 count=136
I1115 15:30:44.180948 2915 Stat.cpp:133] Stat=eval TID=2915 total=0.003 avg=0.003 max=0.003 min=0.003 count=1
I1115 15:30:44.180960 2915 Stat.cpp:133] Stat=mergeRemoteGradSparse TID=3321 total=1.029 avg=0.033 max=0.37 min=0.004 count=31
Stat=mergeRemoteGradSparse TID=3322 total=1.025 avg=0.033 max=0.394 min=0.001 count=31
Stat=mergeRemoteGradSparse TID=3319 total=1.018 avg=0.032 max=0.369 min=0.001 count=31
Stat=mergeRemoteGradSparse TID=3317 total=1.028 avg=0.033 max=0.369 min=0.003 count=31
Stat=mergeRemoteGradSparse TID=3324 total=0.965 avg=0.031 max=0.369 min=0 count=31
Stat=mergeRemoteGradSparse TID=3318 total=1.025 avg=0.033 max=0.39 min=0.001 count=31
Stat=mergeRemoteGradSparse TID=3323 total=1.018 avg=0.032 max=0.369 min=0.003 count=31
Stat=mergeRemoteGradSparse TID=3320 total=0.978 avg=0.031 max=0.37 min=0.001 count=31
I1115 15:30:44.180997 2915 Stat.cpp:133] Stat=sumCost TID=2915 total=0 avg=0 max=0 min=0 count=1
I1115 15:30:44.181008 2915 Stat.cpp:133] Stat=sendAndRecv_dense TID=2915 total=12268.4 avg=12268.4 max=12268.4 min=12268.4 count=1
I1115 15:30:44.181022 2915 Stat.cpp:157] --------------------------------------------------